diff --git a/packages/app/src/electron/ai/__tests__/codex-cli-mcp.test.ts b/packages/app/src/electron/ai/__tests__/codex-cli-mcp.test.ts index 09c39fa1..46ed56e6 100644 --- a/packages/app/src/electron/ai/__tests__/codex-cli-mcp.test.ts +++ b/packages/app/src/electron/ai/__tests__/codex-cli-mcp.test.ts @@ -22,14 +22,12 @@ const mocks = vi.hoisted(() => { model, provider, createCodexAppServer: vi.fn(() => provider), - createSdkMcpServer: vi.fn(() => ({ url: "http://127.0.0.1/mcp" })), tool: vi.fn((definition) => definition), }; }); vi.mock("ai-sdk-provider-codex-cli", () => ({ createCodexAppServer: mocks.createCodexAppServer, - createSdkMcpServer: mocks.createSdkMcpServer, tool: mocks.tool, })); @@ -90,7 +88,14 @@ describe("CodexCliAdapter MCP transport", () => { requestInteraction: vi.fn(async () => ({ approved: false })), }); - const mcpServer = mocks.createSdkMcpServer.mock.results[0].value; + const mcpServer = providerSettings()?.mcpServers?.convera; + expect(mcpServer).toEqual( + expect.objectContaining({ + name: "convera", + _start: expect.any(Function), + _stop: expect.any(Function), + }), + ); expect(mocks.provider).toHaveBeenCalledWith( "gpt-test", expect.objectContaining({ diff --git a/packages/app/src/electron/ai/providers/claude-code.ts b/packages/app/src/electron/ai/providers/claude-code.ts index b4a566bf..742acfad 100644 --- a/packages/app/src/electron/ai/providers/claude-code.ts +++ b/packages/app/src/electron/ai/providers/claude-code.ts @@ -11,6 +11,7 @@ import { resolveLocalModelId, type LocalAiProviderAdapter, } from "../provider-adapter"; +import { toMcpToolResult } from "../tool-result"; import type { LocalAiProviderStatus } from "../types"; export class ClaudeCodeAdapter implements LocalAiProviderAdapter { @@ -52,17 +53,7 @@ export class ClaudeCodeAdapter implements LocalAiProviderAdapter { async (input) => { try { const output = await definition.execute(input); - return { - content: [ - { - type: "text" as const, - text: - typeof output === "string" - ? output - : JSON.stringify(output), - }, - ], - }; + return toMcpToolResult(output); } catch (error) { return { content: [ diff --git a/packages/app/src/electron/ai/providers/codex-cli.ts b/packages/app/src/electron/ai/providers/codex-cli.ts index 3849c855..0de6e12d 100644 --- a/packages/app/src/electron/ai/providers/codex-cli.ts +++ b/packages/app/src/electron/ai/providers/codex-cli.ts @@ -11,6 +11,7 @@ import { type LocalAiProviderAdapter, } from "../provider-adapter"; import type { LocalAiProviderStatus } from "../types"; +import { createCodexMcpServer } from "./codex-mcp-server"; export class CodexCliAdapter implements LocalAiProviderAdapter { readonly id = "codex-cli" as const; @@ -56,8 +57,7 @@ export class CodexCliAdapter implements LocalAiProviderAdapter { context: Parameters[2], ): Promise { await this.ensureProvider(status.executablePath); - const { createSdkMcpServer, tool } = - await importCodexProviderWithZod3Compatibility(); + const { tool } = await importCodexProviderWithZod3Compatibility(); const tools = context.tools.map((definition) => tool({ name: definition.name, @@ -69,7 +69,11 @@ export class CodexCliAdapter implements LocalAiProviderAdapter { ); const mcpServer = tools.length > 0 - ? createSdkMcpServer({ name: "convera", tools }) + ? createCodexMcpServer({ + name: "convera", + tools, + definitions: context.tools, + }) : undefined; const requestApproval = async ( name: string, diff --git a/packages/app/src/electron/ai/providers/codex-mcp-server.test.ts b/packages/app/src/electron/ai/providers/codex-mcp-server.test.ts new file mode 100644 index 00000000..19693cc9 --- /dev/null +++ b/packages/app/src/electron/ai/providers/codex-mcp-server.test.ts @@ -0,0 +1,89 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; +import type { LocalTool } from "ai-sdk-provider-codex-cli"; +import { describe, expect, it, vi } from "vitest"; +import { z } from "zod"; +import type { AgentTool } from "../agent-tools"; +import { createCodexMcpServer } from "./codex-mcp-server"; + +describe("Codex image-capable MCP server", () => { + it("preserves image content and rejects unauthenticated callers", async () => { + const execute = vi.fn(async () => ({ + content: [ + { type: "text" as const, text: "screen" }, + { type: "image" as const, data: "cG5n", mimeType: "image/png" }, + ], + })); + const definition: AgentTool = { + name: "builtin__computer_control", + qualifiedName: "builtin:computer_control", + description: "Capture the screen", + inputSchema: { + type: "object", + properties: { action: { type: "string" } }, + }, + inputShape: { action: z.string() }, + inputValidator: z.object({ action: z.string() }), + execute, + }; + const localTool: LocalTool = { + name: definition.name, + description: definition.description, + inputSchema: definition.inputSchema, + execute, + }; + const server = createCodexMcpServer({ + name: "convera", + tools: [localTool], + definitions: [definition], + }); + + const config = await server._start(); + expect(config.transport).toBe("http"); + if (config.transport !== "http") { + throw new Error("Expected HTTP MCP transport."); + } + + const unauthorized = await fetch(config.url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { name: "test", version: "1.0.0" }, + }, + }), + }); + expect(unauthorized.status).toBe(401); + + const client = new Client({ name: "test", version: "1.0.0" }); + const transport = new StreamableHTTPClientTransport(new URL(config.url), { + requestInit: { + headers: { Authorization: `Bearer ${config.bearerToken}` }, + }, + }); + + try { + await client.connect(transport); + await expect( + client.callTool({ + name: definition.name, + arguments: { action: "screenshot" }, + }), + ).resolves.toEqual({ + content: [ + { type: "text", text: "screen" }, + { type: "image", data: "cG5n", mimeType: "image/png" }, + ], + }); + expect(execute).toHaveBeenCalledWith({ action: "screenshot" }); + } finally { + await client.close(); + await server._stop(); + } + }); +}); diff --git a/packages/app/src/electron/ai/providers/codex-mcp-server.ts b/packages/app/src/electron/ai/providers/codex-mcp-server.ts new file mode 100644 index 00000000..a869cbbb --- /dev/null +++ b/packages/app/src/electron/ai/providers/codex-mcp-server.ts @@ -0,0 +1,176 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import type { ServerResponse } from "node:http"; +import { createServer, type Server } from "node:http"; +import { randomBytes } from "node:crypto"; +import type { LocalTool, SdkMcpServer } from "ai-sdk-provider-codex-cli"; +import type { AgentTool } from "../agent-tools"; +import { toMcpToolResult } from "../tool-result"; + +const SDK_MCP_SERVER_MARKER = Symbol.for( + "ai-sdk-provider-codex-cli.sdkMcpServer", +); + +function sendJson( + response: ServerResponse, + statusCode: number, + body: unknown, +): void { + response.writeHead(statusCode, { "Content-Type": "application/json" }); + response.end(JSON.stringify(body)); +} + +async function closeHttpServer(server: Server): Promise { + if (!server.listening) return; + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +async function waitForResponse(response: ServerResponse): Promise { + if (response.writableFinished || response.destroyed) return; + await new Promise((resolve) => { + response.once("finish", resolve); + response.once("close", resolve); + }); +} + +function createRequestServer( + name: string, + definitions: AgentTool[], +): McpServer { + const server = new McpServer({ name, version: "1.0.0" }); + for (const definition of definitions) { + server.registerTool( + definition.name, + { + description: definition.description, + inputSchema: definition.inputShape, + }, + async (input) => + toMcpToolResult( + await definition.execute(input as Record), + ), + ); + } + return server; +} + +export function createCodexMcpServer(options: { + name: string; + tools: LocalTool[]; + definitions: AgentTool[]; +}): SdkMcpServer { + const bearerToken = randomBytes(32).toString("hex"); + const expectedAuthorization = `Bearer ${bearerToken}`; + let httpServer: Server | undefined; + let startPromise: Promise<{ + transport: "http"; + url: string; + bearerToken: string; + }> | null = null; + let stopPromise: Promise | null = null; + + const sdkServer = { + [SDK_MCP_SERVER_MARKER]: true, + name: options.name, + tools: options.tools, + async _start() { + while (stopPromise) await stopPromise; + if (httpServer?.listening) { + const address = httpServer.address(); + if (address && typeof address !== "string") { + return { + transport: "http" as const, + url: `http://127.0.0.1:${address.port}/mcp`, + bearerToken, + }; + } + } + if (startPromise) return startPromise; + + startPromise = new Promise((resolve, reject) => { + const server = createServer(async (request, response) => { + if (request.url !== "/mcp") { + sendJson(response, 404, { error: "Not found" }); + return; + } + if (request.method !== "POST") { + sendJson(response, 405, { error: "Method not allowed" }); + return; + } + if (request.headers.authorization !== expectedAuthorization) { + response.setHeader("WWW-Authenticate", "Bearer"); + sendJson(response, 401, { error: "Unauthorized" }); + return; + } + + const mcpServer = createRequestServer( + options.name, + options.definitions, + ); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined, + enableJsonResponse: true, + }); + try { + await mcpServer.connect(transport); + await transport.handleRequest(request, response); + await waitForResponse(response); + } catch (error) { + if (!response.headersSent) { + sendJson(response, 500, { + jsonrpc: "2.0", + id: null, + error: { + code: -32603, + message: + error instanceof Error ? error.message : String(error), + }, + }); + } + } finally { + await mcpServer.close(); + } + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + const address = server.address(); + if (!address || typeof address === "string") { + reject(new Error("Failed to resolve local MCP server address.")); + return; + } + httpServer = server; + resolve({ + transport: "http", + url: `http://127.0.0.1:${address.port}/mcp`, + bearerToken, + }); + }); + }); + + try { + return await startPromise; + } finally { + startPromise = null; + } + }, + async _stop() { + if (stopPromise) return stopPromise; + stopPromise = (async () => { + await startPromise?.catch(() => undefined); + const server = httpServer; + httpServer = undefined; + if (server) await closeHttpServer(server); + })(); + try { + await stopPromise; + } finally { + stopPromise = null; + } + }, + }; + + return sdkServer as unknown as SdkMcpServer; +} diff --git a/packages/app/src/electron/ai/tool-result.test.ts b/packages/app/src/electron/ai/tool-result.test.ts new file mode 100644 index 00000000..b6b43867 --- /dev/null +++ b/packages/app/src/electron/ai/tool-result.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { isMcpToolResult, toMcpToolResult } from "./tool-result"; + +describe("tool result conversion", () => { + it("preserves MCP image content", () => { + const result = { + content: [ + { type: "text" as const, text: "screen" }, + { type: "image" as const, data: "cG5n", mimeType: "image/png" }, + ], + }; + + expect(isMcpToolResult(result)).toBe(true); + expect(toMcpToolResult(result)).toBe(result); + }); + + it("converts ordinary tool output to MCP text content", () => { + expect(toMcpToolResult({ success: true })).toEqual({ + content: [{ type: "text", text: '{"success":true}' }], + }); + }); +}); diff --git a/packages/app/src/electron/ai/tool-result.ts b/packages/app/src/electron/ai/tool-result.ts new file mode 100644 index 00000000..4d0e622d --- /dev/null +++ b/packages/app/src/electron/ai/tool-result.ts @@ -0,0 +1,33 @@ +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function isMcpToolResult(value: unknown): value is CallToolResult { + return ( + isRecord(value) && + Array.isArray(value.content) && + value.content.every( + (content) => isRecord(content) && typeof content.type === "string", + ) + ); +} + +export function toMcpToolResult(output: unknown): CallToolResult { + if (isMcpToolResult(output)) { + return output; + } + + return { + content: [ + { + type: "text", + text: + typeof output === "string" + ? output + : (JSON.stringify(output) ?? String(output)), + }, + ], + }; +} diff --git a/packages/app/src/electron/mcp/runtime-catalog.test.ts b/packages/app/src/electron/mcp/runtime-catalog.test.ts index c5e77b07..a49f1051 100644 --- a/packages/app/src/electron/mcp/runtime-catalog.test.ts +++ b/packages/app/src/electron/mcp/runtime-catalog.test.ts @@ -56,6 +56,7 @@ describe("main-process agent tool catalog", () => { const context = createModel.mock.calls[0]?.[2]; expect(context?.tools.map((tool) => tool.qualifiedName)).toEqual([ "builtin:ask_user_input", + "builtin:computer_control", "builtin:execute_command", "builtin:web_fetch", ]); diff --git a/packages/app/src/electron/tools/computer-control.test.ts b/packages/app/src/electron/tools/computer-control.test.ts new file mode 100644 index 00000000..7584b91b --- /dev/null +++ b/packages/app/src/electron/tools/computer-control.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createComputerControl, + type ComputerBitmap, + type ComputerControlDependencies, + type ComputerRobot, +} from "./computer-control"; + +function testRig( + permissionOverrides: Partial< + Awaited> + > = {}, +) { + const bitmap: ComputerBitmap = { + width: 1440, + height: 900, + image: Buffer.alloc(1440 * 900 * 4), + byteWidth: 1440 * 4, + bitsPerPixel: 32, + bytesPerPixel: 4, + }; + const robot: ComputerRobot = { + screen: { capture: vi.fn(() => bitmap) }, + getScreenSize: vi.fn(() => ({ width: 1440, height: 900 })), + getMousePos: vi.fn(() => ({ x: 100, y: 200 })), + moveMouse: vi.fn(), + moveMouseSmooth: vi.fn(), + mouseClick: vi.fn(), + mouseToggle: vi.fn(), + scrollMouse: vi.fn(), + keyTap: vi.fn(), + typeString: vi.fn(), + }; + const dependencies: ComputerControlDependencies = { + getRobot: vi.fn(async () => robot), + bitmapToPng: vi.fn(async () => Buffer.from("png")), + getPermissions: vi.fn(async () => ({ + accessibility: true, + screenRecording: "granted", + ...permissionOverrides, + })), + wait: vi.fn(async () => undefined), + }; + const computer = createComputerControl(dependencies); + const execute = ( + input: Parameters>[0], + ) => + computer.execute!(input, { + toolCallId: "computer-control-test", + messages: [], + }); + + return { bitmap, computer, dependencies, execute, robot }; +} + +describe("computer_control", () => { + it("returns screenshots as MCP image content with coordinate metadata", async () => { + const { execute } = testRig(); + + await expect(execute({ action: "screenshot" })).resolves.toEqual({ + content: [ + { + type: "text", + text: JSON.stringify({ + success: true, + action: "screenshot", + screen: { width: 1440, height: 900 }, + cursor: { x: 100, y: 200 }, + coordinateSpace: "Screenshot top-left is [0, 0].", + }), + }, + { + type: "image", + data: Buffer.from("png").toString("base64"), + mimeType: "image/png", + }, + ], + }); + }); + + it("executes clicks at validated screenshot coordinates", async () => { + const { execute, robot } = testRig(); + + await execute({ action: "double_click", coordinate: [320, 240] }); + + expect(robot.moveMouse).toHaveBeenCalledWith(320, 240); + expect(robot.mouseClick).toHaveBeenCalledWith("left", true); + }); + + it("parses portable key shortcuts", async () => { + const { execute, robot } = testRig(); + + await execute({ action: "key", key: "CMD+SHIFT+P" }); + + expect(robot.keyTap).toHaveBeenCalledWith("p", ["command", "shift"]); + }); + + it("releases the mouse if a drag fails", async () => { + const { execute, robot } = testRig(); + vi.mocked(robot.moveMouseSmooth).mockImplementation(() => { + throw new Error("drag failed"); + }); + + await expect( + execute({ + action: "left_click_drag", + start_coordinate: [10, 20], + coordinate: [30, 40], + }), + ).rejects.toThrow("drag failed"); + expect(robot.mouseToggle).toHaveBeenNthCalledWith(1, "down", "left"); + expect(robot.mouseToggle).toHaveBeenNthCalledWith(2, "up", "left"); + }); + + it("returns actionable validation and permission errors", async () => { + const { execute } = testRig({ accessibility: false }); + + await expect(execute({ action: "left_click" })).rejects.toThrow( + "requires 'coordinate'", + ); + await expect( + execute({ action: "left_click", coordinate: [2000, 10] }), + ).rejects.toThrow("ACCESSIBILITY_PERMISSION_REQUIRED"); + }); +}); diff --git a/packages/app/src/electron/tools/computer-control.ts b/packages/app/src/electron/tools/computer-control.ts new file mode 100644 index 00000000..2964fa4c --- /dev/null +++ b/packages/app/src/electron/tools/computer-control.ts @@ -0,0 +1,387 @@ +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { tool } from "ai"; +import { z } from "zod"; + +type MouseButton = "left" | "right" | "middle"; +type KeyModifier = "alt" | "command" | "control" | "shift"; + +export interface ComputerBitmap { + width: number; + height: number; + image: Buffer; + byteWidth: number; + bitsPerPixel: number; + bytesPerPixel: number; +} + +export interface ComputerRobot { + screen: { + capture( + x?: number, + y?: number, + width?: number, + height?: number, + ): ComputerBitmap; + }; + getScreenSize(): { width: number; height: number }; + getMousePos(): { x: number; y: number }; + moveMouse(x: number, y: number): void; + moveMouseSmooth(x: number, y: number): void; + mouseClick(button?: MouseButton, double?: boolean): void; + mouseToggle(state: "up" | "down", button?: MouseButton): void; + scrollMouse(x: number, y: number): void; + keyTap(key: string, modifier?: KeyModifier | KeyModifier[]): void; + typeString(text: string): void; +} + +export interface ComputerControlDependencies { + getRobot(): Promise; + bitmapToPng(bitmap: ComputerBitmap): Promise; + getPermissions(): Promise<{ + accessibility: boolean | "unsupported"; + screenRecording: string | "unsupported"; + }>; + wait(durationMs: number): Promise; +} + +const coordinateSchema = z + .tuple([z.number().int().nonnegative(), z.number().int().nonnegative()]) + .describe("Absolute [x, y] coordinates in the screenshot coordinate space"); + +const computerControlInput = z.object({ + action: z + .enum([ + "screenshot", + "mouse_move", + "left_click", + "right_click", + "middle_click", + "double_click", + "left_click_drag", + "type", + "key", + "scroll", + "wait", + ]) + .describe("One atomic computer action to execute"), + coordinate: coordinateSchema.optional(), + start_coordinate: coordinateSchema + .optional() + .describe("Drag start [x, y]; required for left_click_drag"), + text: z + .string() + .max(20000) + .optional() + .describe("Text to type; required for the type action"), + key: z + .string() + .max(100) + .optional() + .describe("Key or shortcut such as ENTER, CMD+L, or CTRL+SHIFT+P"), + scroll_x: z + .number() + .int() + .min(-10000) + .max(10000) + .optional() + .describe("Horizontal scroll amount; negative scrolls left"), + scroll_y: z + .number() + .int() + .min(-10000) + .max(10000) + .optional() + .describe("Vertical scroll amount; negative scrolls down"), + duration_ms: z + .number() + .int() + .min(0) + .max(10000) + .optional() + .describe("Wait duration in milliseconds; required for wait"), +}); + +type ComputerControlInput = z.infer; + +const actionRequirements: Partial< + Record> +> = { + mouse_move: ["coordinate"], + left_click: ["coordinate"], + right_click: ["coordinate"], + middle_click: ["coordinate"], + double_click: ["coordinate"], + left_click_drag: ["start_coordinate", "coordinate"], + type: ["text"], + key: ["key"], + wait: ["duration_ms"], +}; + +function validateActionInput(input: ComputerControlInput): void { + for (const field of actionRequirements[input.action] ?? []) { + if (input[field] === undefined) { + throw new Error( + `INVALID_COMPUTER_ACTION: '${input.action}' requires '${field}'. Add the missing field and retry.`, + ); + } + } + + if ( + input.action === "scroll" && + input.scroll_x === undefined && + input.scroll_y === undefined + ) { + throw new Error( + "INVALID_COMPUTER_ACTION: 'scroll' requires scroll_x or scroll_y. Add a non-zero scroll amount and retry.", + ); + } +} + +function validateCoordinate( + coordinate: [number, number], + screenSize: { width: number; height: number }, +): void { + const [x, y] = coordinate; + if (x >= screenSize.width || y >= screenSize.height) { + throw new Error( + `COORDINATE_OUT_OF_BOUNDS: [${x}, ${y}] is outside ${screenSize.width}x${screenSize.height}. Take a new screenshot and retry with coordinates inside it.`, + ); + } +} + +function parseKeyShortcut(value: string): { + key: string; + modifiers?: KeyModifier[]; +} { + const parts = value + .split("+") + .map((part) => part.trim().toLowerCase()) + .filter(Boolean); + const key = parts.pop(); + if (!key) { + throw new Error( + "INVALID_KEY: provide a key such as ENTER, CMD+L, or CTRL+SHIFT+P.", + ); + } + + const aliases: Record = { + alt: "alt", + option: "alt", + cmd: "command", + command: "command", + meta: "command", + ctrl: "control", + control: "control", + shift: "shift", + }; + const modifiers = parts.map((part) => aliases[part]); + const invalidModifier = parts.find((_, index) => !modifiers[index]); + if (invalidModifier) { + throw new Error( + `INVALID_KEY_MODIFIER: '${invalidModifier}' is unsupported. Use CMD, CTRL, ALT/OPTION, or SHIFT.`, + ); + } + + return { + key, + modifiers: modifiers.length ? modifiers : undefined, + }; +} + +function textResult( + input: ComputerControlInput, + robot: ComputerRobot, +): CallToolResult { + const screen = robot.getScreenSize(); + const cursor = robot.getMousePos(); + return { + content: [ + { + type: "text", + text: JSON.stringify({ + success: true, + action: input.action, + screen, + cursor, + }), + }, + ], + }; +} + +async function screenshotResult( + robot: ComputerRobot, + bitmapToPng: ComputerControlDependencies["bitmapToPng"], +): Promise { + const bitmap = robot.screen.capture(); + const png = await bitmapToPng(bitmap); + const cursor = robot.getMousePos(); + return { + content: [ + { + type: "text", + text: JSON.stringify({ + success: true, + action: "screenshot", + screen: { width: bitmap.width, height: bitmap.height }, + cursor, + coordinateSpace: "Screenshot top-left is [0, 0].", + }), + }, + { + type: "image", + data: png.toString("base64"), + mimeType: "image/png", + }, + ], + }; +} + +function assertPermission( + action: ComputerControlInput["action"], + permissions: Awaited< + ReturnType + >, +): void { + if (action === "screenshot") { + if ( + permissions.screenRecording !== "granted" && + permissions.screenRecording !== "unsupported" + ) { + throw new Error( + `SCREEN_RECORDING_PERMISSION_REQUIRED: current status is '${permissions.screenRecording}'. Grant Convera Screen Recording permission in system settings, restart Convera, and retry.`, + ); + } + return; + } + + if (action !== "wait" && permissions.accessibility === false) { + throw new Error( + "ACCESSIBILITY_PERMISSION_REQUIRED: grant Convera Accessibility permission in system settings, restart Convera, and retry.", + ); + } +} + +async function defaultDependencies(): Promise { + return { + getRobot: async () => + (await import("../../shared/robot.js")) + .default as unknown as ComputerRobot, + bitmapToPng: async (bitmap) => { + if ( + bitmap.bitsPerPixel !== 32 || + bitmap.bytesPerPixel !== 4 || + bitmap.byteWidth !== bitmap.width * 4 + ) { + throw new Error( + `UNSUPPORTED_SCREEN_BITMAP: expected packed 32-bit pixels, received ${bitmap.bitsPerPixel}-bit with byte width ${bitmap.byteWidth}.`, + ); + } + const { nativeImage } = await import("electron"); + const image = nativeImage.createFromBitmap(Buffer.from(bitmap.image), { + width: bitmap.width, + height: bitmap.height, + scaleFactor: 1, + }); + const png = image.toPNG(); + if (png.length === 0) { + throw new Error( + "SCREENSHOT_ENCODING_FAILED: Electron could not encode the captured display. Retry after checking Screen Recording permission.", + ); + } + return png; + }, + getPermissions: async () => { + if (process.platform !== "darwin") { + return { + accessibility: "unsupported", + screenRecording: "unsupported", + }; + } + const { systemPreferences } = await import("electron"); + return { + accessibility: systemPreferences.isTrustedAccessibilityClient(false), + screenRecording: systemPreferences.getMediaAccessStatus("screen"), + }; + }, + wait: (durationMs) => + new Promise((resolve) => setTimeout(resolve, durationMs)), + }; +} + +export function createComputerControl( + dependencies?: ComputerControlDependencies, +) { + return tool({ + description: + "Observe and control the user's real desktop with one atomic action. Use screenshot first and again after actions when visual confirmation is needed. Coordinates use the latest screenshot's top-left as [0, 0]. Supported actions are screenshot, mouse_move, left_click, right_click, middle_click, double_click, left_click_drag, type, key, scroll, and wait. Returns JSON state for actions and an MCP image plus screen dimensions for screenshots. Desktop contents may be sensitive and every call requires user approval.", + inputSchema: computerControlInput, + execute: async (input) => { + validateActionInput(input); + const resolvedDependencies = + dependencies ?? (await defaultDependencies()); + const permissions = await resolvedDependencies.getPermissions(); + assertPermission(input.action, permissions); + const robot = await resolvedDependencies.getRobot(); + + if (input.action === "screenshot") { + return screenshotResult(robot, resolvedDependencies.bitmapToPng); + } + + const screenSize = robot.getScreenSize(); + if (input.coordinate) { + validateCoordinate(input.coordinate, screenSize); + } + if (input.start_coordinate) { + validateCoordinate(input.start_coordinate, screenSize); + } + + switch (input.action) { + case "mouse_move": + robot.moveMouse(...input.coordinate!); + break; + case "left_click": + case "right_click": + case "middle_click": + case "double_click": { + robot.moveMouse(...input.coordinate!); + const button = + input.action === "right_click" + ? "right" + : input.action === "middle_click" + ? "middle" + : "left"; + robot.mouseClick(button, input.action === "double_click"); + break; + } + case "left_click_drag": + robot.moveMouse(...input.start_coordinate!); + robot.mouseToggle("down", "left"); + try { + robot.moveMouseSmooth(...input.coordinate!); + } finally { + robot.mouseToggle("up", "left"); + } + break; + case "type": + robot.typeString(input.text!); + break; + case "key": { + const shortcut = parseKeyShortcut(input.key!); + robot.keyTap(shortcut.key, shortcut.modifiers); + break; + } + case "scroll": + robot.scrollMouse(input.scroll_x ?? 0, input.scroll_y ?? 0); + break; + case "wait": + await resolvedDependencies.wait(input.duration_ms!); + break; + } + + return textResult(input, robot); + }, + }); +} + +export const computerControl = createComputerControl(); diff --git a/packages/app/src/electron/tools/index.ts b/packages/app/src/electron/tools/index.ts index 40e1051c..330864a1 100644 --- a/packages/app/src/electron/tools/index.ts +++ b/packages/app/src/electron/tools/index.ts @@ -5,11 +5,13 @@ */ import { askUserInput } from "./ask-user-input"; +import { computerControl } from "./computer-control"; import { executeCommand } from "./execute-command"; import { webFetch } from "./web-fetch"; export const builtinTools = { askUserInput, + computerControl, executeCommand, webFetch, }; @@ -20,6 +22,7 @@ export const builtinTools = { */ export const BUILTIN_TOOLS_REGISTRY = { ask_user_input: askUserInput, + computer_control: computerControl, execute_command: executeCommand, web_fetch: webFetch, } as const; @@ -30,6 +33,11 @@ export const BUILTIN_TOOL_ANNOTATIONS = { destructiveHint: false, openWorldHint: false, }, + computer_control: { + readOnlyHint: false, + destructiveHint: true, + openWorldHint: true, + }, execute_command: { readOnlyHint: false, destructiveHint: true, diff --git a/packages/app/src/renderer/components/settings/pages/general-page.tsx b/packages/app/src/renderer/components/settings/pages/general-page.tsx index 32bfbd53..ea3bea3f 100644 --- a/packages/app/src/renderer/components/settings/pages/general-page.tsx +++ b/packages/app/src/renderer/components/settings/pages/general-page.tsx @@ -1,8 +1,12 @@ import { Button } from "@/renderer/components/ui/button"; import { useLocalAIProviders } from "@/renderer/libs/hooks/use-local-ai-providers"; +import { + DEFAULT_LOCAL_AI_MODEL_ID, + isLocalAIProviderId, +} from "@/renderer/libs/local-ai"; import { useModelConfigStore } from "@/renderer/libs/stores/model-config-store"; import { useSettingsStore } from "@/renderer/libs/stores/settings-store"; -import { Loader2, RotateCcw, Terminal } from "lucide-react"; +import { Check, Loader2, RotateCcw, Terminal } from "lucide-react"; import React, { useCallback, useEffect, useRef } from "react"; export function GeneralSettingsPage() { @@ -12,7 +16,8 @@ export function GeneralSettingsPage() { const saveTimeoutRef = useRef(null); // Model Config state - const { subscribeToModelConfigChanges } = useModelConfigStore(); + const { selectedConfigId, setSelectedModel, subscribeToModelConfigChanges } = + useModelConfigStore(); const { providers, loading: providersLoading } = useLocalAIProviders(); // Settings Store @@ -312,49 +317,76 @@ export function GeneralSettingsPage() { Local AI Providers

- Convera uses your existing Claude Code or Codex CLI login. + Choose the local CLI Convera uses for new chats. Your selection is + stored only on this device.

- {providers.map((provider) => ( -
-
-
- + {providers.map((provider) => { + const isSelected = provider.id === selectedConfigId; + const isAvailable = provider.availability === "available"; + const canSelect = + !providersLoading && + isAvailable && + isLocalAIProviderId(provider.id); + + return ( +
- ))} + + ); + })}
diff --git a/packages/app/src/types/robotjs.d.ts b/packages/app/src/types/robotjs.d.ts index c7bee822..5639ee1e 100644 --- a/packages/app/src/types/robotjs.d.ts +++ b/packages/app/src/types/robotjs.d.ts @@ -1,4 +1,19 @@ declare module "@hurdlegroup/robotjs" { + export interface Bitmap { + width: number; + height: number; + image: Buffer; + byteWidth: number; + bitsPerPixel: number; + bytesPerPixel: number; + colorAt(x: number, y: number): string; + } + + export const screen: { + capture(x?: number, y?: number, width?: number, height?: number): Bitmap; + updateMetrics(): void; + }; + export function keyTap(key: string, modifier?: string | string[]): void; export function keyToggle( key: string, @@ -9,6 +24,7 @@ declare module "@hurdlegroup/robotjs" { export function typeStringDelayed(text: string, cpm: number): void; export function moveMouse(x: number, y: number): void; export function moveMouseSmooth(x: number, y: number): void; + export function dragMouse(x: number, y: number): void; export function mouseClick( button?: "left" | "right" | "middle", double?: boolean, @@ -21,12 +37,6 @@ declare module "@hurdlegroup/robotjs" { export function getMousePos(): { x: number; y: number }; export function getPixelColor(x: number, y: number): string; export function getScreenSize(): { width: number; height: number }; - export function captureScreen( - x?: number, - y?: number, - width?: number, - height?: number, - ): Buffer; export function setKeyboardDelay(ms: number): void; export function setMouseDelay(ms: number): void; }