fix(browser): keep agent browser work visible

apps/desktop/src/ipc/channels.ts

@@ -25,6 +25,7 @@ export const PREVIEW_REVEAL_ELEMENT_CHANNEL = "desktop:preview-reveal-element";
 export const PREVIEW_SET_VIEWPORT_CHANNEL = "desktop:preview-set-viewport";
 export const PREVIEW_SET_NAVIGATION_POLICY_CHANNEL = "desktop:preview-set-navigation-policy";
 export const PREVIEW_NAVIGATION_BLOCKED_CHANNEL = "desktop:preview-navigation-blocked";
+export const PREVIEW_USER_CONTROL_CHANNEL = "desktop:preview-user-control";
 export const PREVIEW_CLEAR_BROWSING_DATA_CHANNEL = "desktop:preview-clear-browsing-data";
 export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";
 export const SET_THEME_CHANNEL = "desktop:set-theme";

apps/desktop/src/preload.ts

@@ -144,6 +144,17 @@ contextBridge.exposeInMainWorld("desktopBridge", {
       ipcRenderer.removeListener(IpcChannels.PREVIEW_NAVIGATION_BLOCKED_CHANNEL, wrappedListener);
     };
   },
+  onPreviewUserControl: (listener) => {
+    const wrappedListener = (_event: Electron.IpcRendererEvent, control: unknown) => {
+      if (typeof control !== "object" || control === null) return;
+      listener(control as Parameters<typeof listener>[0]);
+    };
+
+    ipcRenderer.on(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, wrappedListener);
+    return () => {
+      ipcRenderer.removeListener(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, wrappedListener);
+    };
+  },
   previewPickElement: (input) =>
     ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, input),
   previewCancelPick: (input) => ipcRenderer.invoke(IpcChannels.PREVIEW_CANCEL_PICK_CHANNEL, input),

apps/desktop/src/preview/PreviewAutomation.ts

@@ -26,6 +26,7 @@ import type {
   DesktopPreviewNetworkFailure,
   DesktopPreviewMoveInput,
   DesktopPreviewNavigationBlocked,
+  DesktopPreviewUserControl,
   DesktopPreviewPressInput,
   DesktopPreviewScrollInput,
   DesktopPreviewSnapshot,
@@ -279,6 +280,20 @@ export const make = Effect.sync(function PreviewAutomationMake() {
     }
   };
 
+  const reportUserControl = (contents: WebContents) => {
+    const payload: DesktopPreviewUserControl = { webContentsId: contents.id };
+    const embedder = contents.hostWebContents;
+    if (embedder !== null && !embedder.isDestroyed()) {
+      embedder.send(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, payload);
+      return;
+    }
+    for (const window of BrowserWindow.getAllWindows()) {
+      if (!window.isDestroyed()) {
+        window.webContents.send(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, payload);
+      }
+    }
+  };
+
   /**
    * Arms the guest's own navigations against the guest's allowlist.
    *
@@ -858,6 +873,7 @@ export const make = Effect.sync(function PreviewAutomationMake() {
     const onBeforeInput = (_event: unknown, input: { type?: string }) => {
       if (input.type === "keyDown" && !consumeExpectedAgentInput("keyDown")) {
         tab.controlEpoch += 1;
+        reportUserControl(contents);
       }
     };
     const onBeforeMouse = (_event: unknown, input: { type?: string }) => {
@@ -867,7 +883,10 @@ export const make = Effect.sync(function PreviewAutomationMake() {
           : input.type === "mouseWheel"
             ? "mouseWheel"
             : null;
-      if (kind !== null && !consumeExpectedAgentInput(kind)) tab.controlEpoch += 1;
+      if (kind !== null && !consumeExpectedAgentInput(kind)) {
+        tab.controlEpoch += 1;
+        reportUserControl(contents);
+      }
     };
     contents.on("before-input-event", onBeforeInput);
     contents.on("before-mouse-event", onBeforeMouse);

apps/server/src/mcp/browserTools.ts

@@ -96,7 +96,7 @@ export const BrowserTabsTool = readsOnly(
 export const BrowserOpenTabTool = changesThePage(
   Tool.make("browser_open_tab", {
     description:
-      "Create a browser tab and pin your future browser actions to it. Give a URL to load it immediately. Set background true to leave the user's visible tab alone; otherwise the new tab is brought to the front.",
+      "Create a browser tab owned by this agent and pin future browser actions to it. Give a URL to load it immediately. If the agent had to open a closed browser panel, the new tab is shown even when background is true; otherwise background leaves the user's visible tab alone.",
     parameters: PreviewAutomationOpenTabInputSchema,
     success: PreviewAutomationStatusSchema,
     failure: PreviewAutomationErrorSchema,
@@ -107,7 +107,7 @@ export const BrowserOpenTabTool = changesThePage(
 export const BrowserCloseTabTool = changesThePage(
   Tool.make("browser_close_tab", {
     description:
-      "Close a browser tab. Omit tabId to close your pinned tab, or use a stable id from browser_tabs. Returns the remaining tabs.",
+      "Close one of this agent's own browser tabs. Omit tabId to close the pinned tab, or use an owned stable id from browser_tabs. The result reports the closed page, remaining tabs, and whether the browser panel stayed open.",
     parameters: PreviewAutomationCloseTabInputSchema,
     success: PreviewAutomationTabsSchema,
     failure: PreviewAutomationErrorSchema,

apps/web/src/browserPanelStore.test.ts

@@ -49,7 +49,10 @@ describe("nextActiveTabId", () => {
 
 describe("background tabs", () => {
   beforeEach(() => {
-    useBrowserPanelStore.setState({ browserStateByThreadKey: {} });
+    useBrowserPanelStore.setState({
+      browserStateByThreadKey: {},
+      browserOwnershipByThreadKey: {},
+    });
   });
 
   it("creates a live tab without changing the user's active tab", () => {
@@ -64,6 +67,138 @@ describe("background tabs", () => {
     expect(next.activeTabId).toBe(original.activeTabId);
     expect(next.tabs.find((tab) => tab.id === openedId)?.url).toBe("http://localhost:5173/");
   });
+
+  it("shows an agent background tab when the agent opened a closed browser", () => {
+    const store = useBrowserPanelStore.getState();
+
+    store.openBrowserForAgent(THREAD_REF, "agent-a");
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+      background: true,
+    });
+
+    const next = selectThreadBrowserState(
+      useBrowserPanelStore.getState().browserStateByThreadKey,
+      THREAD_REF,
+    );
+    expect(next.open).toBe(true);
+    expect(next.activeTabId).toBe(openedId);
+    expect(next.tabs).toHaveLength(1);
+    expect(next.tabs[0]?.url).toBe("http://localhost:5173/");
+  });
+
+  it("keeps a user-opened browser focused while adding an agent background tab", () => {
+    const store = useBrowserPanelStore.getState();
+    store.setBrowserOpen(THREAD_REF, true);
+    const original = selectThreadBrowserState(
+      useBrowserPanelStore.getState().browserStateByThreadKey,
+      THREAD_REF,
+    );
+
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+      background: true,
+    });
+
+    const next = selectThreadBrowserState(
+      useBrowserPanelStore.getState().browserStateByThreadKey,
+      THREAD_REF,
+    );
+    expect(next.open).toBe(true);
+    expect(next.activeTabId).toBe(original.activeTabId);
+    expect(next.tabs.map((tab) => tab.id)).toContain(openedId);
+  });
+
+  it("closes an agent-opened panel when the final agent tab closes", () => {
+    const store = useBrowserPanelStore.getState();
+    store.openBrowserForAgent(THREAD_REF, "agent-a");
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+    });
+
+    store.closeAgentTab(THREAD_REF, "agent-a", openedId);
+
+    const next = selectThreadBrowserState(
+      useBrowserPanelStore.getState().browserStateByThreadKey,
+      THREAD_REF,
+    );
+    expect(next.open).toBe(false);
+    expect(next.tabs).toHaveLength(1);
+    expect(next.tabs[0]?.url).toBeNull();
+  });
+
+  it("keeps the panel open when the user took control before the agent tab closed", () => {
+    const store = useBrowserPanelStore.getState();
+    store.openBrowserForAgent(THREAD_REF, "agent-a");
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+    });
+    store.markBrowserUserControlled(THREAD_REF);
+
+    store.closeAgentTab(THREAD_REF, "agent-a", openedId);
+
+    const next = selectThreadBrowserState(
+      useBrowserPanelStore.getState().browserStateByThreadKey,
+      THREAD_REF,
+    );
+    expect(next.open).toBe(true);
+    expect(next.tabs).toHaveLength(1);
+    expect(next.tabs[0]?.url).toBeNull();
+  });
+
+  it("keeps an agent-opened panel until every agent closes its own tabs", () => {
+    const store = useBrowserPanelStore.getState();
+    store.openBrowserForAgent(THREAD_REF, "agent-a");
+    const first = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/a",
+    });
+    const second = store.openAgentTab(THREAD_REF, "agent-b", {
+      url: "http://localhost:5173/b",
+      background: true,
+    });
+
+    expect(store.closeAgentTab(THREAD_REF, "agent-a", first)).toEqual({
+      closed: true,
+      panelOpen: true,
+    });
+    expect(store.closeAgentTab(THREAD_REF, "agent-b", second)).toEqual({
+      closed: true,
+      panelOpen: false,
+    });
+  });
+
+  it("does not let one agent close another agent's tab", () => {
+    const store = useBrowserPanelStore.getState();
+    store.setBrowserOpen(THREAD_REF, true);
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+    });
+
+    expect(store.closeAgentTab(THREAD_REF, "agent-b", openedId)).toEqual({
+      closed: false,
+      panelOpen: true,
+    });
+    expect(
+      selectThreadBrowserState(
+        useBrowserPanelStore.getState().browserStateByThreadKey,
+        THREAD_REF,
+      ).tabs.some((tab) => tab.id === openedId),
+    ).toBe(true);
+  });
+
+  it("keeps a user-opened panel after its agent tab closes", () => {
+    const store = useBrowserPanelStore.getState();
+    store.setBrowserOpen(THREAD_REF, true);
+    const openedId = store.openAgentTab(THREAD_REF, "agent-a", {
+      url: "http://localhost:5173/",
+      background: true,
+    });
+
+    expect(store.closeAgentTab(THREAD_REF, "agent-a", openedId)).toEqual({
+      closed: true,
+      panelOpen: true,
+    });
+  });
 });
 
 describe("steppedZoom", () => {