feat(app): connect local AI tools through MCP

packages/app/automation/driver.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, it, vi } from "vitest";
+import { ConveraDriver } from "./driver.js";
+
+describe("ConveraDriver click", () => {
+  it("returns the pre-click snapshot when the clicked element disappears", async () => {
+    let exists = true;
+    const click = vi.fn(async () => {
+      exists = false;
+    });
+    const element = {
+      attributes: [{ name: "role", value: "option" }],
+      click,
+      doubleClick: vi.fn(),
+      getTagName: vi.fn(async () => "button"),
+      getText: vi.fn(async () => "Alpha"),
+      getValue: vi.fn(async () => null),
+      isClickable: vi.fn(async () => exists),
+      isDisplayed: vi.fn(async () => exists),
+      isEnabled: vi.fn(async () => true),
+      isExisting: vi.fn(async () => exists),
+      waitForDisplayed: vi.fn(async () => undefined),
+      waitForExist: vi.fn(async () => undefined),
+    };
+    const browser = {
+      $: vi.fn(async () => element),
+      execute: vi.fn(
+        async (
+          operation: (node: typeof element) => unknown,
+          node: typeof element,
+        ) => operation(node),
+      ),
+      waitUntil: vi.fn(async (condition: () => Promise<boolean>) => {
+        if (!(await condition())) throw new Error("condition not met");
+        return true;
+      }),
+    };
+    const driver = new ConveraDriver();
+    Reflect.set(driver, "browser", browser);
+
+    await expect(driver.click('[role="option"]')).resolves.toMatchObject({
+      selector: '[role="option"]',
+      tag: "button",
+      text: "Alpha",
+      displayed: true,
+      enabled: true,
+      clickable: true,
+      attributes: { role: "option" },
+      action: "click",
+      completed: true,
+    });
+    expect(click).toHaveBeenCalledOnce();
+    expect(browser.$).toHaveBeenCalledTimes(2);
+    expect(element.isExisting).toHaveBeenCalledTimes(1);
+  });
+});

packages/app/automation/driver.ts

@@ -547,9 +547,14 @@ export class ConveraDriver {
 
   async click(selector: string, double = false) {
     const element = await this.readyElement(selector, true);
+    const before = await this.inspectElement(selector);
     if (double) await element.doubleClick();
     else await element.click();
-    return this.inspectElement(selector);
+    return {
+      ...before,
+      action: double ? "double_click" : "click",
+      completed: true,
+    };
   }
 
   async hover(selector: string) {

packages/app/src/electro-bridge/ipc/local-ai-context.test.ts

@@ -86,6 +86,7 @@ function createRuntime(
     })),
     startChat: vi.fn(),
     abort: vi.fn(() => true),
+    respondToInteraction: vi.fn(() => false),
     ...overrides,
   };
 }
@@ -116,25 +117,25 @@ describe("local AI IPC", () => {
     listener?.(
       {},
       {
-        type: "delta",
+        type: "ui-message",
         requestId: "request-2",
-        text: "ignore",
+        chunk: { type: "text-delta", id: "text-1", delta: "ignore" },
       },
     );
     listener?.(
       {},
       {
-        type: "delta",
+        type: "ui-message",
         requestId: "request-1",
-        text: "hello",
+        chunk: { type: "text-delta", id: "text-1", delta: "hello" },
       },
     );
 
     expect(callback).toHaveBeenCalledOnce();
     expect(callback).toHaveBeenCalledWith({
-      type: "delta",
+      type: "ui-message",
       requestId: "request-1",
-      text: "hello",
+      chunk: { type: "text-delta", id: "text-1", delta: "hello" },
     });
 
     unsubscribe();
@@ -151,9 +152,9 @@ describe("local AI IPC", () => {
     const runtime = createRuntime({
       startChat: vi.fn((_request, emit) => {
         emit({
-          type: "delta",
+          type: "ui-message",
           requestId: "runtime-cannot-change-owner",
-          text: "hello",
+          chunk: { type: "text-delta", id: "text-1", delta: "hello" },
         });
         emit({
           type: "finish",
@@ -193,9 +194,9 @@ describe("local AI IPC", () => {
       {
         channel: LOCAL_AI_CHANNELS.EVENT,
         event: {
-          type: "delta",
+          type: "ui-message",
           requestId: "request-1",
-          text: "hello",
+          chunk: { type: "text-delta", id: "text-1", delta: "hello" },
         },
       },
       {
@@ -251,6 +252,78 @@ describe("local AI IPC", () => {
     expect(runtime.startChat).not.toHaveBeenCalled();
   });
 
+  it("accepts interaction responses only from the active request owner", async () => {
+    const allowedSender = new FakeWebContents(1);
+    const otherSender = new FakeWebContents(2);
+    const runtime = createRuntime({
+      startChat: vi.fn(() => new Promise<void>(() => undefined)),
+      respondToInteraction: vi.fn(() => true),
+    });
+    const { handlers, ipc } = createMainIPC();
+    setupLocalAIIPC(
+      {
+        runtime,
+        getAllowedWebContents: () => allowedSender as never,
+      },
+      ipc as never,
+    );
+    const start = handlers.get(LOCAL_AI_CHANNELS.START_CHAT);
+    const respond = handlers.get(LOCAL_AI_CHANNELS.RESPOND_INTERACTION);
+
+    start?.(createEvent(allowedSender), {
+      requestId: "request-1",
+      providerId: "claude-code",
+      messages: [{ role: "user", content: "hello" }],
+    });
+
+    await expect(
+      respond?.(createEvent(allowedSender), "request-1", "interaction-1", {
+        approved: true,
+      }),
+    ).resolves.toEqual({
+      success: true,
+      data: { accepted: true },
+    });
+    expect(runtime.respondToInteraction).toHaveBeenCalledWith(
+      "request-1",
+      "interaction-1",
+      { approved: true },
+    );
+
+    await expect(
+      respond?.(createEvent(otherSender), "request-1", "interaction-1", {
+        approved: true,
+      }),
+    ).resolves.toMatchObject({
+      success: false,
+      error: { code: "LOCAL_AI_FORBIDDEN" },
+    });
+  });
+
+  it("rejects malformed interaction responses", async () => {
+    const sender = new FakeWebContents(1);
+    const runtime = createRuntime();
+    const { handlers, ipc } = createMainIPC();
+    setupLocalAIIPC(
+      {
+        runtime,
+        getAllowedWebContents: () => sender as never,
+      },
+      ipc as never,
+    );
+    const respond = handlers.get(LOCAL_AI_CHANNELS.RESPOND_INTERACTION);
+
+    await expect(
+      respond?.(createEvent(sender), "request-1", "interaction-1", {
+        approved: "yes",
+      }),
+    ).resolves.toMatchObject({
+      success: false,
+      error: { code: "LOCAL_AI_INVALID_REQUEST" },
+    });
+    expect(runtime.respondToInteraction).not.toHaveBeenCalled();
+  });
+
   it("aborts active work when its webContents is destroyed", async () => {
     const sender = new FakeWebContents(1);
     let resolveChat: (() => void) | undefined;

packages/app/src/electro-bridge/ipc/local-ai-context.ts

@@ -1,6 +1,7 @@
 import type {
   ILocalAIAPI,
   LocalAIChatRequest,
+  LocalAIInteractionResponse,
   LocalAIProviderStatus,
   LocalAIResult,
   LocalAIRuntimeService,
@@ -23,6 +24,7 @@ export const LOCAL_AI_CHANNELS = {
   GET_PROVIDER_STATUS: "local-ai:get-provider-status",
   START_CHAT: "local-ai:start-chat",
   ABORT: "local-ai:abort",
+  RESPOND_INTERACTION: "local-ai:respond-interaction",
   EVENT: "local-ai:event",
 } as const;
 
@@ -45,6 +47,7 @@ const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;
 const ALLOWED_PROVIDER_IDS = new Set(["claude-code", "codex-cli"]);
 const MAX_MESSAGE_CHARS = 200_000;
 const MAX_REQUEST_CHARS = 1_000_000;
+const MAX_INTERACTION_RESPONSE_CHARS = 20_000;
 
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
@@ -130,6 +133,28 @@ function validateRequest(request: unknown): request is LocalAIChatRequest {
   });
 }
 
+function validateInteractionResponse(
+  response: unknown,
+): response is LocalAIInteractionResponse {
+  if (!isRecord(response)) return false;
+
+  const keys = Object.keys(response);
+  if (
+    keys.length === 0 ||
+    keys.some((key) => key !== "approved" && key !== "value")
+  ) {
+    return false;
+  }
+
+  return (
+    (response.approved === undefined ||
+      typeof response.approved === "boolean") &&
+    (response.value === undefined ||
+      (typeof response.value === "string" &&
+        response.value.length <= MAX_INTERACTION_RESPONSE_CHARS))
+  );
+}
+
 function failure<T>(error: unknown): LocalAIResult<T> {
   return { success: false, error: serializeLocalAIError(error) };
 }
@@ -305,13 +330,7 @@ export function setupLocalAIIPC(
                 requestId: request.requestId,
                 error: serializeLocalAIError(runtimeEvent.error),
               }
-            : runtimeEvent.type === "tool" && runtimeEvent.error
-              ? {
-                  ...runtimeEvent,
-                  requestId: request.requestId,
-                  error: serializeLocalAIError(runtimeEvent.error),
-                }
-              : { ...runtimeEvent, requestId: request.requestId };
+            : { ...runtimeEvent, requestId: request.requestId };
 
         try {
           sender.send(LOCAL_AI_CHANNELS.EVENT, streamEvent);
@@ -354,6 +373,57 @@ export function setupLocalAIIPC(
     },
   );
 
+  mainIPC.handle(
+    LOCAL_AI_CHANNELS.RESPOND_INTERACTION,
+    async (
+      event,
+      requestId: unknown,
+      interactionId: unknown,
+      response: unknown,
+    ): Promise<LocalAIResult<{ accepted: boolean }>> => {
+      if (!ensureSender(event)) {
+        return failure(
+          createError("IPC sender is not allowed", "LOCAL_AI_FORBIDDEN"),
+        );
+      }
+      if (!options.runtime) return failure(runtimeUnavailable());
+      if (
+        typeof requestId !== "string" ||
+        !REQUEST_ID_PATTERN.test(requestId) ||
+        typeof interactionId !== "string" ||
+        !REQUEST_ID_PATTERN.test(interactionId) ||
+        !validateInteractionResponse(response)
+      ) {
+        return failure(
+          createError(
+            "Invalid local AI interaction response",
+            "LOCAL_AI_INVALID_REQUEST",
+          ),
+        );
+      }
+
+      const active = activeRequests.get(requestId);
+      if (!active || active.sender !== event.sender) {
+        return { success: true, data: { accepted: false } };
+      }
+
+      try {
+        return {
+          success: true,
+          data: {
+            accepted: await options.runtime.respondToInteraction(
+              requestId,
+              interactionId,
+              response,
+            ),
+          },
+        };
+      } catch (error) {
+        return failure(error);
+      }
+    },
+  );
+
   mainIPC.handle(
     LOCAL_AI_CHANNELS.ABORT,
     async (
@@ -433,6 +503,13 @@ export function createLocalAIAPI(
       rendererIPC.invoke(LOCAL_AI_CHANNELS.START_CHAT, request),
     abort: (requestId) =>
       rendererIPC.invoke(LOCAL_AI_CHANNELS.ABORT, requestId),
+    respondToInteraction: (requestId, interactionId, response) =>
+      rendererIPC.invoke(
+        LOCAL_AI_CHANNELS.RESPOND_INTERACTION,
+        requestId,
+        interactionId,
+        response,
+      ),
     onEvent: (requestId, callback) => {
       const handler = (_event: unknown, event: LocalAIStreamEvent) => {
         if (event.requestId === requestId) callback(event);