diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 6b9d63a85..90c455417 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -25,6 +25,7 @@ export const PREVIEW_REVEAL_ELEMENT_CHANNEL = "desktop:preview-reveal-element"; export const PREVIEW_SET_VIEWPORT_CHANNEL = "desktop:preview-set-viewport"; export const PREVIEW_SET_NAVIGATION_POLICY_CHANNEL = "desktop:preview-set-navigation-policy"; export const PREVIEW_NAVIGATION_BLOCKED_CHANNEL = "desktop:preview-navigation-blocked"; +export const PREVIEW_USER_CONTROL_CHANNEL = "desktop:preview-user-control"; export const PREVIEW_CLEAR_BROWSING_DATA_CHANNEL = "desktop:preview-clear-browsing-data"; export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache"; export const SET_THEME_CHANNEL = "desktop:set-theme"; diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 391a83f19..9bae667ba 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -144,6 +144,17 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.removeListener(IpcChannels.PREVIEW_NAVIGATION_BLOCKED_CHANNEL, wrappedListener); }; }, + onPreviewUserControl: (listener) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, control: unknown) => { + if (typeof control !== "object" || control === null) return; + listener(control as Parameters[0]); + }; + + ipcRenderer.on(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, wrappedListener); + return () => { + ipcRenderer.removeListener(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, wrappedListener); + }; + }, previewPickElement: (input) => ipcRenderer.invoke(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, input), previewCancelPick: (input) => ipcRenderer.invoke(IpcChannels.PREVIEW_CANCEL_PICK_CHANNEL, input), diff --git a/apps/desktop/src/preview/PreviewAutomation.ts b/apps/desktop/src/preview/PreviewAutomation.ts index 00732cc2a..258bf4d35 100644 --- a/apps/desktop/src/preview/PreviewAutomation.ts +++ b/apps/desktop/src/preview/PreviewAutomation.ts @@ -26,6 +26,7 @@ import type { DesktopPreviewNetworkFailure, DesktopPreviewMoveInput, DesktopPreviewNavigationBlocked, + DesktopPreviewUserControl, DesktopPreviewPressInput, DesktopPreviewScrollInput, DesktopPreviewSnapshot, @@ -279,6 +280,20 @@ export const make = Effect.sync(function PreviewAutomationMake() { } }; + const reportUserControl = (contents: WebContents) => { + const payload: DesktopPreviewUserControl = { webContentsId: contents.id }; + const embedder = contents.hostWebContents; + if (embedder !== null && !embedder.isDestroyed()) { + embedder.send(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, payload); + return; + } + for (const window of BrowserWindow.getAllWindows()) { + if (!window.isDestroyed()) { + window.webContents.send(IpcChannels.PREVIEW_USER_CONTROL_CHANNEL, payload); + } + } + }; + /** * Arms the guest's own navigations against the guest's allowlist. * @@ -858,6 +873,7 @@ export const make = Effect.sync(function PreviewAutomationMake() { const onBeforeInput = (_event: unknown, input: { type?: string }) => { if (input.type === "keyDown" && !consumeExpectedAgentInput("keyDown")) { tab.controlEpoch += 1; + reportUserControl(contents); } }; const onBeforeMouse = (_event: unknown, input: { type?: string }) => { @@ -867,7 +883,10 @@ export const make = Effect.sync(function PreviewAutomationMake() { : input.type === "mouseWheel" ? "mouseWheel" : null; - if (kind !== null && !consumeExpectedAgentInput(kind)) tab.controlEpoch += 1; + if (kind !== null && !consumeExpectedAgentInput(kind)) { + tab.controlEpoch += 1; + reportUserControl(contents); + } }; contents.on("before-input-event", onBeforeInput); contents.on("before-mouse-event", onBeforeMouse); diff --git a/apps/server/src/mcp/browserTools.ts b/apps/server/src/mcp/browserTools.ts index 343eb4e41..1750edf41 100644 --- a/apps/server/src/mcp/browserTools.ts +++ b/apps/server/src/mcp/browserTools.ts @@ -96,7 +96,7 @@ export const BrowserTabsTool = readsOnly( export const BrowserOpenTabTool = changesThePage( Tool.make("browser_open_tab", { description: - "Create a browser tab and pin your future browser actions to it. Give a URL to load it immediately. Set background true to leave the user's visible tab alone; otherwise the new tab is brought to the front.", + "Create a browser tab owned by this agent and pin future browser actions to it. Give a URL to load it immediately. If the agent had to open a closed browser panel, the new tab is shown even when background is true; otherwise background leaves the user's visible tab alone.", parameters: PreviewAutomationOpenTabInputSchema, success: PreviewAutomationStatusSchema, failure: PreviewAutomationErrorSchema, @@ -107,7 +107,7 @@ export const BrowserOpenTabTool = changesThePage( export const BrowserCloseTabTool = changesThePage( Tool.make("browser_close_tab", { description: - "Close a browser tab. Omit tabId to close your pinned tab, or use a stable id from browser_tabs. Returns the remaining tabs.", + "Close one of this agent's own browser tabs. Omit tabId to close the pinned tab, or use an owned stable id from browser_tabs. The result reports the closed page, remaining tabs, and whether the browser panel stayed open.", parameters: PreviewAutomationCloseTabInputSchema, success: PreviewAutomationTabsSchema, failure: PreviewAutomationErrorSchema, diff --git a/apps/web/src/browserPanelStore.test.ts b/apps/web/src/browserPanelStore.test.ts index bda9ca8d2..634238f01 100644 --- a/apps/web/src/browserPanelStore.test.ts +++ b/apps/web/src/browserPanelStore.test.ts @@ -49,7 +49,10 @@ describe("nextActiveTabId", () => { describe("background tabs", () => { beforeEach(() => { - useBrowserPanelStore.setState({ browserStateByThreadKey: {} }); + useBrowserPanelStore.setState({ + browserStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, + }); }); it("creates a live tab without changing the user's active tab", () => { @@ -64,6 +67,138 @@ describe("background tabs", () => { expect(next.activeTabId).toBe(original.activeTabId); expect(next.tabs.find((tab) => tab.id === openedId)?.url).toBe("http://localhost:5173/"); }); + + it("shows an agent background tab when the agent opened a closed browser", () => { + const store = useBrowserPanelStore.getState(); + + store.openBrowserForAgent(THREAD_REF, "agent-a"); + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + background: true, + }); + + const next = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ); + expect(next.open).toBe(true); + expect(next.activeTabId).toBe(openedId); + expect(next.tabs).toHaveLength(1); + expect(next.tabs[0]?.url).toBe("http://localhost:5173/"); + }); + + it("keeps a user-opened browser focused while adding an agent background tab", () => { + const store = useBrowserPanelStore.getState(); + store.setBrowserOpen(THREAD_REF, true); + const original = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ); + + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + background: true, + }); + + const next = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ); + expect(next.open).toBe(true); + expect(next.activeTabId).toBe(original.activeTabId); + expect(next.tabs.map((tab) => tab.id)).toContain(openedId); + }); + + it("closes an agent-opened panel when the final agent tab closes", () => { + const store = useBrowserPanelStore.getState(); + store.openBrowserForAgent(THREAD_REF, "agent-a"); + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + }); + + store.closeAgentTab(THREAD_REF, "agent-a", openedId); + + const next = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ); + expect(next.open).toBe(false); + expect(next.tabs).toHaveLength(1); + expect(next.tabs[0]?.url).toBeNull(); + }); + + it("keeps the panel open when the user took control before the agent tab closed", () => { + const store = useBrowserPanelStore.getState(); + store.openBrowserForAgent(THREAD_REF, "agent-a"); + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + }); + store.markBrowserUserControlled(THREAD_REF); + + store.closeAgentTab(THREAD_REF, "agent-a", openedId); + + const next = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ); + expect(next.open).toBe(true); + expect(next.tabs).toHaveLength(1); + expect(next.tabs[0]?.url).toBeNull(); + }); + + it("keeps an agent-opened panel until every agent closes its own tabs", () => { + const store = useBrowserPanelStore.getState(); + store.openBrowserForAgent(THREAD_REF, "agent-a"); + const first = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/a", + }); + const second = store.openAgentTab(THREAD_REF, "agent-b", { + url: "http://localhost:5173/b", + background: true, + }); + + expect(store.closeAgentTab(THREAD_REF, "agent-a", first)).toEqual({ + closed: true, + panelOpen: true, + }); + expect(store.closeAgentTab(THREAD_REF, "agent-b", second)).toEqual({ + closed: true, + panelOpen: false, + }); + }); + + it("does not let one agent close another agent's tab", () => { + const store = useBrowserPanelStore.getState(); + store.setBrowserOpen(THREAD_REF, true); + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + }); + + expect(store.closeAgentTab(THREAD_REF, "agent-b", openedId)).toEqual({ + closed: false, + panelOpen: true, + }); + expect( + selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + THREAD_REF, + ).tabs.some((tab) => tab.id === openedId), + ).toBe(true); + }); + + it("keeps a user-opened panel after its agent tab closes", () => { + const store = useBrowserPanelStore.getState(); + store.setBrowserOpen(THREAD_REF, true); + const openedId = store.openAgentTab(THREAD_REF, "agent-a", { + url: "http://localhost:5173/", + background: true, + }); + + expect(store.closeAgentTab(THREAD_REF, "agent-a", openedId)).toEqual({ + closed: true, + panelOpen: true, + }); + }); }); describe("steppedZoom", () => { diff --git a/apps/web/src/browserPanelStore.ts b/apps/web/src/browserPanelStore.ts index 5cd424c76..f2314dcc8 100644 --- a/apps/web/src/browserPanelStore.ts +++ b/apps/web/src/browserPanelStore.ts @@ -58,6 +58,15 @@ export interface ThreadBrowserState { activeTabId: string; } +export interface ThreadBrowserOwnership { + /** Who caused the currently open panel to appear. Runtime-only. */ + openedBy: "user" | "agent" | null; + /** Once true, the agent must stop treating the panel as its private workspace. */ + userControlled: boolean; + /** Tab id to the agent that created (or inherited) it. Runtime-only. */ + agentOwnerByTabId: Readonly>; +} + const BROWSER_PANEL_STORAGE_KEY = "threadlines:browser-panel:v2"; /** @@ -116,6 +125,12 @@ const EMPTY_THREAD_STATE: ThreadBrowserState = Object.freeze({ activeTabId: "", }); +const EMPTY_BROWSER_OWNERSHIP: ThreadBrowserOwnership = Object.freeze({ + openedBy: null, + userControlled: false, + agentOwnerByTabId: {}, +}); + /** Chrome's range, so the familiar steps land on familiar numbers. */ export const ZOOM_STEPS = [0.5, 0.67, 0.75, 0.9, 1, 1.1, 1.25, 1.5, 1.75, 2] as const; @@ -128,7 +143,7 @@ export function clampZoomFactor(factor: number): number { /** The next step up or down, so zooming lands on round values rather than drifting. */ export function steppedZoom(current: number, direction: 1 | -1): number { - const steps = direction === 1 ? ZOOM_STEPS : [...ZOOM_STEPS].reverse(); + const steps = direction === 1 ? ZOOM_STEPS : ZOOM_STEPS.toReversed(); const next = steps.find((step) => direction === 1 ? step > current + 0.001 : step < current - 0.001, ); @@ -346,6 +361,7 @@ export interface PendingBrowserApproval { interface BrowserPanelStoreState { browserStateByThreadKey: Record; agentStateByThreadKey: Record; + browserOwnershipByThreadKey: Record; pendingApprovalByThreadKey: Record; splitChatFraction: number; /** Hides the chat so the page gets the whole centre; the split is remembered. */ @@ -358,6 +374,7 @@ interface BrowserPanelStoreState { deviceToolbarOpen: boolean; appearance: BrowserAppearance; setBrowserOpen: (threadRef: ScopedThreadRef, open: boolean) => void; + openBrowserForAgent: (threadRef: ScopedThreadRef, agentId: string) => void; toggleBrowserOpen: (threadRef: ScopedThreadRef) => void; openTab: (threadRef: ScopedThreadRef, activate?: boolean) => string; /** @@ -369,6 +386,17 @@ interface BrowserPanelStoreState { */ openTabWithUrl: (threadRef: ScopedThreadRef, url: string, activate?: boolean) => string; closeTab: (threadRef: ScopedThreadRef, tabId: string) => void; + openAgentTab: ( + threadRef: ScopedThreadRef, + agentId: string, + input: { url?: string | null; background?: boolean | undefined }, + ) => string; + closeAgentTab: ( + threadRef: ScopedThreadRef, + agentId: string, + tabId: string, + ) => { closed: boolean; panelOpen: boolean }; + markBrowserUserControlled: (threadRef: ScopedThreadRef) => void; selectTab: (threadRef: ScopedThreadRef, tabId: string) => void; setTabUrl: (threadRef: ScopedThreadRef, tabId: string, url: string) => void; /** @@ -424,6 +452,18 @@ function updateAgentState( return { agentStateByThreadKey: { ...state.agentStateByThreadKey, [key]: update(current) } }; } +function updateOwnership( + state: BrowserPanelStoreState, + threadRef: ScopedThreadRef, + update: (current: ThreadBrowserOwnership) => ThreadBrowserOwnership, +): Pick { + const key = scopedThreadKey(threadRef); + const current = state.browserOwnershipByThreadKey[key] ?? EMPTY_BROWSER_OWNERSHIP; + return { + browserOwnershipByThreadKey: { ...state.browserOwnershipByThreadKey, [key]: update(current) }, + }; +} + function updateTab( current: ThreadBrowserState, tabId: string, @@ -435,22 +475,102 @@ function updateTab( }; } +function assignAgentOwner( + ownership: ThreadBrowserOwnership, + tabId: string, + agentId: string, +): ThreadBrowserOwnership { + return { + ...ownership, + agentOwnerByTabId: { ...ownership.agentOwnerByTabId, [tabId]: agentId }, + }; +} + +function removeAgentOwner( + ownership: ThreadBrowserOwnership, + tabId: string, +): ThreadBrowserOwnership { + const { [tabId]: _removed, ...agentOwnerByTabId } = ownership.agentOwnerByTabId; + return { + ...ownership, + agentOwnerByTabId, + }; +} + +function shouldForceAgentTabActive(ownership: ThreadBrowserOwnership): boolean { + return ownership.openedBy === "agent" && !ownership.userControlled; +} + +function isOnlyEmptyTab(state: ThreadBrowserState): boolean { + return ( + state.tabs.length === 1 && + state.activeTabId === state.tabs[0]?.id && + state.tabs[0]?.url === null + ); +} + export const useBrowserPanelStore = create()( persist( (set) => ({ browserStateByThreadKey: {}, agentStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, pendingApprovalByThreadKey: {}, splitChatFraction: DEFAULT_BROWSER_SPLIT_CHAT_FRACTION, expanded: false, deviceToolbarOpen: false, appearance: "system", setBrowserOpen: (threadRef, open) => - set((state) => updateThread(state, threadRef, (current) => ({ ...current, open }))), + set((state) => ({ + ...updateThread(state, threadRef, (current) => ({ ...current, open })), + ...updateOwnership(state, threadRef, () => + open + ? { + openedBy: "user", + userControlled: true, + agentOwnerByTabId: + state.browserOwnershipByThreadKey[scopedThreadKey(threadRef)] + ?.agentOwnerByTabId ?? {}, + } + : EMPTY_BROWSER_OWNERSHIP, + ), + })), + openBrowserForAgent: (threadRef, agentId) => + set((state) => { + const key = scopedThreadKey(threadRef); + const current = state.browserStateByThreadKey[key] ?? DEFAULT_THREAD_STATE; + const ownership = state.browserOwnershipByThreadKey[key] ?? EMPTY_BROWSER_OWNERSHIP; + return { + ...updateThread(state, threadRef, (thread) => ({ ...thread, open: true })), + ...updateOwnership(state, threadRef, () => + current.open + ? ownership.openedBy === null + ? { ...ownership, openedBy: "user", userControlled: true } + : ownership + : { + openedBy: "agent", + userControlled: false, + agentOwnerByTabId: isOnlyEmptyTab(current) + ? { [current.activeTabId]: agentId } + : {}, + }, + ), + }; + }), toggleBrowserOpen: (threadRef) => - set((state) => - updateThread(state, threadRef, (current) => ({ ...current, open: !current.open })), - ), + set((state) => { + const key = scopedThreadKey(threadRef); + const current = state.browserStateByThreadKey[key] ?? DEFAULT_THREAD_STATE; + const nextOpen = !current.open; + return { + ...updateThread(state, threadRef, () => ({ ...current, open: nextOpen })), + ...updateOwnership(state, threadRef, () => + nextOpen + ? { openedBy: "user", userControlled: true, agentOwnerByTabId: {} } + : EMPTY_BROWSER_OWNERSHIP, + ), + }; + }), openTab: (threadRef, activate = true) => { const tab = makeBrowserTab(); set((state) => @@ -479,8 +599,8 @@ export const useBrowserPanelStore = create()( return tab.id; }, closeTab: (threadRef, tabId) => - set((state) => - updateThread(state, threadRef, (current) => { + set((state) => ({ + ...updateThread(state, threadRef, (current) => { const nextActive = nextActiveTabId(current.tabs, tabId, current.activeTabId); if (nextActive === null) { // Never leave the panel with no tab at all: closing the last one @@ -494,6 +614,98 @@ export const useBrowserPanelStore = create()( activeTabId: nextActive, }; }), + ...updateOwnership(state, threadRef, (current) => removeAgentOwner(current, tabId)), + })), + openAgentTab: (threadRef, agentId, input) => { + const tab = { ...makeBrowserTab(), url: input.url ?? null }; + set((state) => { + const key = scopedThreadKey(threadRef); + const ownership = state.browserOwnershipByThreadKey[key] ?? EMPTY_BROWSER_OWNERSHIP; + const forceActive = shouldForceAgentTabActive(ownership); + const activate = forceActive || input.background !== true; + return { + ...updateThread(state, threadRef, (current) => { + const replacingBlank = forceActive && isOnlyEmptyTab(current); + return { + ...current, + tabs: replacingBlank ? [tab] : [...current.tabs, tab], + activeTabId: activate ? tab.id : current.activeTabId, + }; + }), + ...updateOwnership(state, threadRef, (current) => { + const replacingBlank = + forceActive && + isOnlyEmptyTab(state.browserStateByThreadKey[key] ?? DEFAULT_THREAD_STATE); + const withoutReplaced = replacingBlank + ? removeAgentOwner( + current, + (state.browserStateByThreadKey[key] ?? DEFAULT_THREAD_STATE).activeTabId, + ) + : current; + return assignAgentOwner(withoutReplaced, tab.id, agentId); + }), + visitedUrls: + tab.url === null + ? state.visitedUrls + : rememberVisit(state.visitedUrls, tab.url, Date.now()), + }; + }); + return tab.id; + }, + closeAgentTab: (threadRef, agentId, tabId) => { + let result = { closed: false, panelOpen: true }; + set((state) => { + const key = scopedThreadKey(threadRef); + const ownership = state.browserOwnershipByThreadKey[key] ?? EMPTY_BROWSER_OWNERSHIP; + const browser = state.browserStateByThreadKey[key] ?? DEFAULT_THREAD_STATE; + if ( + ownership.agentOwnerByTabId[tabId] !== agentId || + !browser.tabs.some((tab) => tab.id === tabId) + ) { + result = { closed: false, panelOpen: browser.open }; + return state; + } + const nextOwnership = removeAgentOwner(ownership, tabId); + const shouldClosePanel = + ownership.openedBy === "agent" && + !ownership.userControlled && + Object.keys(nextOwnership.agentOwnerByTabId).length === 0; + const threadUpdate = updateThread(state, threadRef, (current) => { + if (shouldClosePanel) { + return { ...DEFAULT_THREAD_STATE, open: false }; + } + const nextActive = nextActiveTabId(current.tabs, tabId, current.activeTabId); + if (nextActive === null) { + const replacement = makeBrowserTab(); + return { ...current, tabs: [replacement], activeTabId: replacement.id }; + } + return { + ...current, + tabs: current.tabs.filter((tab) => tab.id !== tabId), + activeTabId: nextActive, + }; + }); + result = { closed: true, panelOpen: !shouldClosePanel && browser.open }; + return { + ...threadUpdate, + ...updateOwnership(state, threadRef, (current) => + shouldClosePanel ? EMPTY_BROWSER_OWNERSHIP : removeAgentOwner(current, tabId), + ), + }; + }); + return result; + }, + markBrowserUserControlled: (threadRef) => + set((state) => + updateOwnership(state, threadRef, (current) => + current.openedBy === null + ? { + openedBy: "user", + userControlled: true, + agentOwnerByTabId: current.agentOwnerByTabId, + } + : { ...current, userControlled: true }, + ), ), selectTab: (threadRef, tabId) => set((state) => @@ -591,6 +803,14 @@ export function selectThreadAgentState( return agentStateByThreadKey[scopedThreadKey(threadRef)] ?? EMPTY_AGENT_STATE; } +export function selectThreadBrowserOwnership( + ownershipByThreadKey: Record, + threadRef: ScopedThreadRef | null, +): ThreadBrowserOwnership { + if (threadRef === null) return EMPTY_BROWSER_OWNERSHIP; + return ownershipByThreadKey[scopedThreadKey(threadRef)] ?? EMPTY_BROWSER_OWNERSHIP; +} + export function selectPendingBrowserApproval( pendingApprovalByThreadKey: Record, threadRef: ScopedThreadRef | null, diff --git a/apps/web/src/components/ChatView.browser.tsx b/apps/web/src/components/ChatView.browser.tsx index db376574b..fa0c15b2f 100644 --- a/apps/web/src/components/ChatView.browser.tsx +++ b/apps/web/src/components/ChatView.browser.tsx @@ -6,6 +6,8 @@ import { ORCHESTRATION_WS_METHODS, EnvironmentId, type EnvironmentApi, + type DesktopPreviewUserControl, + type DesktopPreviewTarget, type MessageId, type OrchestrationEvent, type PreviewAutomationRequest, @@ -6112,7 +6114,7 @@ describe("ChatView timeline estimator parity (full app)", () => { } }); - it("opens a closed browser panel for an arriving automation request", async () => { + it("shows an agent background tab when automation opens a closed browser", async () => { // The agent asks for the page while the panel is shut. It used to be told // there was no browser; now the request is what opens one. const tab = makeBrowserTab(); @@ -6121,6 +6123,7 @@ describe("ChatView timeline estimator parity (full app)", () => { [THREAD_KEY]: { open: false, tabs: [tab], activeTabId: tab.id }, }, agentStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, }); let deliver: ((request: PreviewAutomationRequest) => void) | null = null; @@ -6148,9 +6151,21 @@ describe("ChatView timeline estimator parity (full app)", () => { } as unknown as EnvironmentApi["previewAutomation"], }), ); + let userControlListener: ((event: { webContentsId: number }) => void) | null = null; window.desktopBridge = { - previewStatus: () => - Promise.resolve({ url: "http://localhost:5173/", title: "Preview", loading: false }), + previewStatus: (input: DesktopPreviewTarget) => + Promise.resolve({ + url: + input.webContentsId === 43 ? "http://localhost:5173/manual" : "http://localhost:5173/", + title: "Preview", + loading: false, + }), + onPreviewUserControl: (listener: (event: DesktopPreviewUserControl) => void) => { + userControlListener = listener; + return () => { + userControlListener = null; + }; + }, } as unknown as NonNullable; const screen = await render(); @@ -6165,8 +6180,9 @@ describe("ChatView timeline estimator parity (full app)", () => { deliver!({ requestId: "req-auto-open", - operation: "status", - input: {}, + agentId: "agent-browser", + operation: "openTab", + input: { url: "http://localhost:5173/manual", background: true }, } as unknown as PreviewAutomationRequest); // The panel opens on its own... @@ -6179,27 +6195,75 @@ describe("ChatView timeline estimator parity (full app)", () => { { timeout: 4_000, interval: 16 }, ); - // ...and once its page is up, the operation the agent asked for lands. + // The initial blank guest lets the queued operation begin. registerPreviewWebview(THREAD_REF, tab.id, { getWebContentsId: () => 42, loadURL: () => Promise.resolve(), getBoundingClientRect: () => ({ width: 900, height: 600 }) as DOMRect, }); + let openedTabId = ""; + await vi.waitFor( + () => { + const state = useBrowserPanelStore.getState().browserStateByThreadKey[THREAD_KEY]; + expect(state?.tabs).toHaveLength(1); + expect(state?.tabs[0]?.url).toBe("http://localhost:5173/manual"); + expect(state?.activeTabId).toBe(state?.tabs[0]?.id); + openedTabId = state?.activeTabId ?? ""; + expect(openedTabId).not.toBe(""); + }, + { timeout: 4_000, interval: 16 }, + ); + + registerPreviewWebview(THREAD_REF, openedTabId, { + getWebContentsId: () => 43, + loadURL: () => Promise.resolve(), + getBoundingClientRect: () => ({ width: 900, height: 600 }) as DOMRect, + }); + await vi.waitFor( () => { expect(responses).toHaveLength(1); expect(responses[0]?.error, "the request should not report a missing browser").toBe( undefined, ); - expect(responses[0]?.result).toMatchObject({ url: "http://localhost:5173/" }); + expect(responses[0]?.result).toMatchObject({ + tabId: openedTabId, + url: "http://localhost:5173/manual", + }); + }, + { timeout: 8_000, interval: 16 }, + ); + + const notifyUserControl = userControlListener as + | ((event: DesktopPreviewUserControl) => void) + | null; + notifyUserControl?.({ webContentsId: 43 }); + deliver!({ + requestId: "req-auto-close-after-takeover", + agentId: "agent-browser", + operation: "closeTab", + input: { tabId: openedTabId }, + } as unknown as PreviewAutomationRequest); + + await vi.waitFor( + () => { + expect(responses).toHaveLength(2); + expect(responses[1]?.result).toMatchObject({ panelOpen: true }); + expect(useBrowserPanelStore.getState().browserStateByThreadKey[THREAD_KEY]?.open).toBe( + true, + ); }, { timeout: 8_000, interval: 16 }, ); } finally { screen.unmount(); resetPreviewWebviewsForTests(); - useBrowserPanelStore.setState({ browserStateByThreadKey: {}, agentStateByThreadKey: {} }); + useBrowserPanelStore.setState({ + browserStateByThreadKey: {}, + agentStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, + }); Reflect.deleteProperty(window, "desktopBridge"); __resetEnvironmentApiOverridesForTests(); } diff --git a/apps/web/src/components/ThreadTerminalDrawer.browser.tsx b/apps/web/src/components/ThreadTerminalDrawer.browser.tsx index 2485d84cd..8666fd6b6 100644 --- a/apps/web/src/components/ThreadTerminalDrawer.browser.tsx +++ b/apps/web/src/components/ThreadTerminalDrawer.browser.tsx @@ -382,7 +382,11 @@ describe("TerminalViewport", () => { linkProviderRef.current = null; terminalBufferLinesRef.current = []; useStore.setState({ activeEnvironmentId: null, environmentStateById: {} }); - useBrowserPanelStore.setState({ browserStateByThreadKey: {}, visitedUrls: [] }); + useBrowserPanelStore.setState({ + browserStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, + visitedUrls: [], + }); useTerminalStateStore.setState({ terminalStateByThreadKey: {}, terminalLaunchContextByThreadKey: {}, diff --git a/apps/web/src/components/browser/AgentActivityLine.browser.tsx b/apps/web/src/components/browser/AgentActivityLine.browser.tsx index a2b5fc692..150fd22d8 100644 --- a/apps/web/src/components/browser/AgentActivityLine.browser.tsx +++ b/apps/web/src/components/browser/AgentActivityLine.browser.tsx @@ -21,6 +21,7 @@ function seedBrowserPanel(viewport: { width: number | null; height: number | nul [THREAD_KEY]: { open: true, tabs: [tab], activeTabId: tab.id }, }, agentStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, pendingApprovalByThreadKey: {}, deviceToolbarOpen: true, }); @@ -30,6 +31,7 @@ function resetBrowserPanel() { useBrowserPanelStore.setState({ browserStateByThreadKey: {}, agentStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, pendingApprovalByThreadKey: {}, deviceToolbarOpen: false, }); diff --git a/apps/web/src/components/browser/BrowserPanel.tsx b/apps/web/src/components/browser/BrowserPanel.tsx index 5bf4c43da..dac240bdd 100644 --- a/apps/web/src/components/browser/BrowserPanel.tsx +++ b/apps/web/src/components/browser/BrowserPanel.tsx @@ -43,6 +43,7 @@ import { selectActiveTab, selectPendingBrowserApproval, selectThreadAgentState, + selectThreadBrowserOwnership, selectThreadBrowserState, steppedZoom, useBrowserPanelStore, @@ -146,11 +147,17 @@ export function BrowserPanel({ const browserState = useBrowserPanelStore((store) => selectThreadBrowserState(store.browserStateByThreadKey, threadRef), ); + const browserOwnership = useBrowserPanelStore((store) => + selectThreadBrowserOwnership(store.browserOwnershipByThreadKey, threadRef), + ); const setTabUrl = useBrowserPanelStore((store) => store.setTabUrl); const setTabViewport = useBrowserPanelStore((store) => store.setTabViewport); const openTab = useBrowserPanelStore((store) => store.openTab); const closeTab = useBrowserPanelStore((store) => store.closeTab); const selectTab = useBrowserPanelStore((store) => store.selectTab); + const markBrowserUserControlled = useBrowserPanelStore( + (store) => store.markBrowserUserControlled, + ); const expanded = useBrowserPanelStore((store) => store.expanded); const toggleExpanded = useBrowserPanelStore((store) => store.toggleExpanded); const deviceToolbarOpen = useBrowserPanelStore((store) => store.deviceToolbarOpen); @@ -659,6 +666,9 @@ export function BrowserPanel({ style={{ flex: `${flexGrow} 1 0%` }} data-testid="browser-panel" aria-label="Browser preview" + onPointerDownCapture={() => markBrowserUserControlled(threadRef)} + onKeyDownCapture={() => markBrowserUserControlled(threadRef)} + onWheelCapture={() => markBrowserUserControlled(threadRef)} >
{/* Tabs scroll; everything after this stays put. ScrollArea keeps the @@ -681,9 +691,9 @@ export function BrowserPanel({ isActive={tab.id === activeTabId} closable agentState={ - tab.id !== agentTabId + browserOwnership.agentOwnerByTabId[tab.id] === undefined ? "none" - : agentActivity?.phase === "running" + : tab.id === agentTabId && agentActivity?.phase === "running" ? "working" : "pinned" } diff --git a/apps/web/src/components/browser/PreviewAutomationMount.tsx b/apps/web/src/components/browser/PreviewAutomationMount.tsx index 18dc63247..fac45c04a 100644 --- a/apps/web/src/components/browser/PreviewAutomationMount.tsx +++ b/apps/web/src/components/browser/PreviewAutomationMount.tsx @@ -64,9 +64,12 @@ export function PreviewAutomationMount({ projectId?: ProjectId | null; }) { const setBrowserOpen = useBrowserPanelStore((store) => store.setBrowserOpen); - const openTab = useBrowserPanelStore((store) => store.openTab); - const openTabWithUrl = useBrowserPanelStore((store) => store.openTabWithUrl); - const closeTab = useBrowserPanelStore((store) => store.closeTab); + const openBrowserForAgent = useBrowserPanelStore((store) => store.openBrowserForAgent); + const openAgentTab = useBrowserPanelStore((store) => store.openAgentTab); + const closeAgentTab = useBrowserPanelStore((store) => store.closeAgentTab); + const markBrowserUserControlled = useBrowserPanelStore( + (store) => store.markBrowserUserControlled, + ); const setAgentTab = useBrowserPanelStore((store) => store.setAgentTab); const setAgentPoint = useBrowserPanelStore((store) => store.setAgentPoint); const setAgentActivity = useBrowserPanelStore((store) => store.setAgentActivity); @@ -143,6 +146,16 @@ export function PreviewAutomationMount({ }); }, [attachedGuests, setBrowserOpen, setPendingBrowserApproval, threadRef]); + useEffect(() => { + const subscribe = window.desktopBridge?.onPreviewUserControl; + if (subscribe === undefined) return; + return subscribe((control) => { + if (attachedGuests().some((entry) => entry.webContentsId === control.webContentsId)) { + markBrowserUserControlled(threadRef); + } + }); + }, [attachedGuests, markBrowserUserControlled, threadRef]); + /** * Reads the world at the moment the agent acts, not at the moment this * component rendered: an operation may arrive many renders after the host @@ -192,6 +205,11 @@ export function PreviewAutomationMount({ webContentsId: webview === null ? null : attachedWebContentsId(webview), onAgentPoint: (point) => setAgentPoint(threadRef, point), onAgentActivity: (activity) => setAgentActivity(threadRef, activity), + onUserTakeover: () => markBrowserUserControlled(threadRef), + panelOpen: () => { + const current = useBrowserPanelStore.getState(); + return selectThreadBrowserState(current.browserStateByThreadKey, threadRef).open; + }, openTab: async (input) => { const normalized = input.url === undefined ? null : normalizePreviewUrl(input.url); if (input.url !== undefined && normalized === null) { @@ -200,7 +218,9 @@ export function PreviewAutomationMount({ if (normalized !== null) { const host = new URL(normalized).hostname; if (!isBrowserHostApproved(host, approvedDomains)) { - const openedId = openTab(threadRef, input.background !== true); + const openedId = openAgentTab(threadRef, request.agentId, { + background: input.background, + }); agentTabPins.set(key, openedId); setAgentTab(threadRef, openedId); setPendingBrowserApproval(threadRef, { @@ -215,22 +235,30 @@ export function PreviewAutomationMount({ ); } } - const activate = input.background !== true; - const openedId = - normalized === null - ? openTab(threadRef, activate) - : openTabWithUrl(threadRef, normalized, activate); + const openedId = openAgentTab(threadRef, request.agentId, { + url: normalized, + background: input.background, + }); agentTabPins.set(key, openedId); setAgentTab(threadRef, openedId); return waitForTab(openedId); }, closeTab: async (closingTabId) => { - if (closingTabId === null || !browserState.tabs.some((tab) => tab.id === closingTabId)) { + const closingTab = browserState.tabs.find((tab) => tab.id === closingTabId); + if (closingTabId === null || closingTab === undefined) { throw new Error("The browser tab to close does not exist."); } - closeTab(threadRef, closingTabId); + const closeResult = closeAgentTab(threadRef, request.agentId, closingTabId); + if (!closeResult.closed) { + throw new Error("You can only close a browser tab opened by this agent."); + } if (agentTabPins.get(key) === closingTabId) agentTabPins.delete(key); if (tabId === closingTabId) setAgentTab(threadRef, null); + return { + id: closingTab.id, + title: closingTab.title ?? "", + url: closingTab.url ?? "", + }; }, selectTab: async (input) => { const chosen = @@ -306,9 +334,9 @@ export function PreviewAutomationMount({ }, [ approvedDomains, - closeTab, - openTab, - openTabWithUrl, + closeAgentTab, + markBrowserUserControlled, + openAgentTab, selectTab, setAgentActivity, setAgentPoint, @@ -331,7 +359,7 @@ export function PreviewAutomationMount({ const store = useBrowserPanelStore.getState(); const browserState = selectThreadBrowserState(store.browserStateByThreadKey, threadRef); if (!browserState.open) { - setBrowserOpen(threadRef, true); + openBrowserForAgent(threadRef, request.agentId); } await waitForPreviewWebview({ resolve: () => { @@ -358,7 +386,7 @@ export function PreviewAutomationMount({ // error it has always used for one, so a panel that never came up reads the // same as a panel with nothing loaded. }, - [setBrowserOpen, threadRef], + [openBrowserForAgent, threadRef], ); // Not the module-level `isElectron` snapshot: the bridge is what this needs, diff --git a/apps/web/src/components/browser/openInBrowserPanel.test.ts b/apps/web/src/components/browser/openInBrowserPanel.test.ts index 683ceed06..18aeb66b5 100644 --- a/apps/web/src/components/browser/openInBrowserPanel.test.ts +++ b/apps/web/src/components/browser/openInBrowserPanel.test.ts @@ -56,7 +56,11 @@ function panelState() { describe("openUrlInBrowserPanel", () => { beforeEach(() => { useStore.setState({ activeEnvironmentId: null, environmentStateById: {} }); - useBrowserPanelStore.setState({ browserStateByThreadKey: {}, visitedUrls: [] }); + useBrowserPanelStore.setState({ + browserStateByThreadKey: {}, + browserOwnershipByThreadKey: {}, + visitedUrls: [], + }); useComposerDraftStore.setState({ draftThreadsByThreadKey: {} }); resetPreviewWebviewsForTests(); }); diff --git a/apps/web/src/components/browser/previewAutomationHost.test.ts b/apps/web/src/components/browser/previewAutomationHost.test.ts index 7fd20fb40..7d06db813 100644 --- a/apps/web/src/components/browser/previewAutomationHost.test.ts +++ b/apps/web/src/components/browser/previewAutomationHost.test.ts @@ -59,18 +59,34 @@ describe("createPreviewAutomationHandler", () => { it("reports human takeover of the tab as an interrupted action", async () => { let controlEpoch = 0; - const handle = handlerFor({ - previewClick: () => { - controlEpoch += 1; - return Promise.resolve({ x: 10, y: 20 }); - }, - previewStatus: () => - Promise.resolve({ url: "http://x/", title: "X", loading: false, controlEpoch }) as never, - }); + let takeoverCount = 0; + const handle = createPreviewAutomationHandler( + { + previewClick: () => { + controlEpoch += 1; + return Promise.resolve({ x: 10, y: 20 }); + }, + previewStatus: () => + Promise.resolve({ url: "http://x/", title: "X", loading: false, controlEpoch }) as never, + } as unknown as DesktopBridge, + () => ({ + webContentsId: 42, + navigate: () => Promise.resolve(), + viewport: () => ({ width: 800, height: 600 }), + onAgentPoint: () => {}, + tabs: () => [], + selectTab: () => {}, + onAgentActivity: () => {}, + onUserTakeover: () => { + takeoverCount += 1; + }, + }), + ); const response = await handle(request("click", { target: { ref: "e1" } })); expect(response.error).toContain("user took control"); + expect(takeoverCount).toBe(1); }); it("sends the operation's input to the bridge along with the tab it acts on", async () => { @@ -218,6 +234,7 @@ describe("createPreviewAutomationHandler", () => { const response = await handle(request("tabs", {})); expect(response.result).toEqual({ + panelOpen: true, tabs: [ { id: "tab-fixture", @@ -237,6 +254,51 @@ describe("createPreviewAutomationHandler", () => { }); }); + it("reports the closed page and whether cleanup closed the panel", async () => { + let open = true; + let tabs = [ + { + id: "tab-agent", + title: "Manual", + url: "https://example.com/manual", + active: true, + agent: true, + }, + ]; + const handle = createPreviewAutomationHandler({} as DesktopBridge, () => ({ + tabId: "tab-agent", + webContentsId: 42, + navigate: () => Promise.resolve(), + viewport: () => ({ width: 800, height: 600 }), + onAgentPoint: () => {}, + onAgentActivity: () => {}, + selectTab: () => {}, + tabs: () => tabs, + panelOpen: () => open, + closeTab: async () => { + tabs = []; + open = false; + return { + id: "tab-agent", + title: "Manual", + url: "https://example.com/manual", + }; + }, + })); + + const response = await handle(request("closeTab")); + + expect(response.result).toEqual({ + tabs: [], + panelOpen: false, + closedTab: { + id: "tab-agent", + title: "Manual", + url: "https://example.com/manual", + }, + }); + }); + it("wraps an evaluate result so an array is not a validation failure", async () => { // The expression has already run by the time the result is checked, so a // bare array used to \"fail\" after mutating the page. diff --git a/apps/web/src/components/browser/previewAutomationHost.ts b/apps/web/src/components/browser/previewAutomationHost.ts index 6b98bbb27..07a5f81a2 100644 --- a/apps/web/src/components/browser/previewAutomationHost.ts +++ b/apps/web/src/components/browser/previewAutomationHost.ts @@ -82,13 +82,17 @@ export interface PreviewAutomationHostTarget { active: boolean; agent: boolean; }>; + /** Whether the panel is still visible after a tab lifecycle operation. */ + readonly panelOpen?: (() => boolean) | undefined; /** Create and resolve a tab for this agent session. */ readonly openTab?: (input: { url?: string | undefined; background?: boolean | undefined; }) => Promise; /** Close a tab and return the listing that remains. */ - readonly closeTab?: (tabId: string | null) => Promise; + readonly closeTab?: ( + tabId: string | null, + ) => Promise<{ id: string; title: string; url: string } | void>; /** Pin this agent to a tab, optionally without moving the user's view. */ readonly selectTab: (input: { tabId?: string | undefined; @@ -97,6 +101,8 @@ export interface PreviewAutomationHostTarget { }) => Promise | PreviewAutomationHostTarget | void; /** What the agent is doing, in words, for the line under the toolbar. */ readonly onAgentActivity: (activity: AgentActivity) => void; + /** The tab changed under the agent because the user acted while it was running. */ + readonly onUserTakeover?: (() => void) | undefined; } /** @@ -283,6 +289,7 @@ export function createPreviewAutomationHandler( ? await bridge.previewStatus({ webContentsId: target.webContentsId as number }) : null; if (before !== null && after !== null && before.controlEpoch !== after.controlEpoch) { + target.onUserTakeover?.(); throw new Error( "The browser action was interrupted because the user took control of this tab.", ); @@ -377,7 +384,7 @@ async function dispatch( return toStatus(target.tabId ?? "", await call(bridge.previewStatus, {}), target.viewport()); } case "tabs": - return { tabs: target.tabs() }; + return { tabs: target.tabs(), panelOpen: target.panelOpen?.() ?? true }; case "openTab": { if (target.openTab === undefined) throw new Error("This build cannot open browser tabs."); const opened = await target.openTab(input); @@ -388,10 +395,22 @@ async function dispatch( opened.viewport(), ); } - case "closeTab": + case "closeTab": { if (target.closeTab === undefined) throw new Error("This build cannot close browser tabs."); - await target.closeTab(typeof input.tabId === "string" ? input.tabId : (target.tabId ?? null)); - return { tabs: target.tabs() }; + const closingTabId = typeof input.tabId === "string" ? input.tabId : (target.tabId ?? null); + const beforeClose = target.tabs().find((tab) => tab.id === closingTabId); + const reportedClosedTab = await target.closeTab(closingTabId); + const closedTab = + reportedClosedTab ?? + (beforeClose === undefined + ? undefined + : { id: beforeClose.id, title: beforeClose.title, url: beforeClose.url }); + return { + tabs: target.tabs(), + panelOpen: target.panelOpen?.() ?? true, + ...(closedTab === undefined ? {} : { closedTab }), + }; + } case "selectTab": { const selected = await target.selectTab(input); if (selected === undefined) throw new Error("The browser tab could not be selected."); diff --git a/apps/web/src/session-logic.test.ts b/apps/web/src/session-logic.test.ts index dd5f589ac..afb595165 100644 --- a/apps/web/src/session-logic.test.ts +++ b/apps/web/src/session-logic.test.ts @@ -409,9 +409,9 @@ describe("deriveThreadSubagentHistory", () => { updatedAt: "2026-08-11T19:34:59.000Z", }; const [turnPreparing, turnStarted, spawn, wait] = codexSpawnActivities(); - const collabSpawn = structuredClone(spawn) as typeof spawn; + const collabSpawn = structuredClone(spawn!) as NonNullable; const collabItem = ( - (collabSpawn?.payload as { data: { item: Record } }).data as { + (collabSpawn.payload as { data: { item: Record } }).data as { item: Record; } ).item; @@ -3054,6 +3054,100 @@ describe("deriveWorkLogEntries", () => { }); }); + it("collapses Threadlines browser work into a durable verification receipt", () => { + const entries = deriveWorkLogEntries([ + makeActivity({ + id: "browser-open-tab", + sequence: 1, + kind: "tool.completed", + summary: "MCP tool call", + payload: { + itemType: "mcp_tool_call", + title: "MCP tool call", + data: { + item: { + server: "browser", + tool: "browser_open_tab", + arguments: { + url: "https://docs.example.com/manual", + background: true, + }, + result: { + structuredContent: { + tabId: "tab-agent-1", + url: "https://docs.example.com/manual", + title: "Manual", + }, + }, + }, + }, + }, + }), + makeActivity({ + id: "browser-wait", + sequence: 2, + kind: "tool.completed", + summary: "MCP tool call", + payload: { + itemType: "mcp_tool_call", + title: "MCP tool call", + data: { + item: { + server: "browser", + tool: "browser_wait_for", + arguments: { tabId: "tab-agent-1", text: "AFP-100" }, + result: { + structuredContent: { + tabId: "tab-agent-1", + url: "https://docs.example.com/manual#verified", + title: "Manual", + }, + }, + }, + }, + }, + }), + makeActivity({ + id: "browser-close-tab", + sequence: 3, + kind: "tool.completed", + summary: "MCP tool call", + payload: { + itemType: "mcp_tool_call", + title: "MCP tool call", + data: { + item: { + server: "browser", + tool: "browser_close_tab", + arguments: { + tabId: "tab-agent-1", + }, + result: { + structuredContent: { + tabs: [], + panelOpen: false, + closedTab: { + id: "tab-agent-1", + url: "https://docs.example.com/manual#verified", + title: "Manual", + }, + }, + }, + }, + }, + }, + }), + ]); + + expect(entries).toHaveLength(1); + expect(entries[0]).toMatchObject({ + toolTitle: "Browser receipt", + detail: + "Opened https://docs.example.com/manual · Final https://docs.example.com/manual#verified · Verified address, load state, and page content · Closed tab and browser", + executionState: "completed", + }); + }); + it("labels Browser skill activity that is routed through node_repl.js", () => { const entries = deriveWorkLogEntries([ makeActivity({ diff --git a/apps/web/src/session-logic.ts b/apps/web/src/session-logic.ts index 49acb329d..f82e828e0 100644 --- a/apps/web/src/session-logic.ts +++ b/apps/web/src/session-logic.ts @@ -173,6 +173,16 @@ export type ActiveModelFallbackState = ModelFallbackState; interface DerivedWorkLogEntry extends WorkLogEntry { activityKind: OrchestrationThreadActivity["kind"]; collapseKey?: string; + browserReceipt?: BrowserReceipt; +} + +interface BrowserReceipt { + tabId: string; + openedUrl: string | null; + finalUrl: string | null; + verifications: ReadonlyArray; + tabClosed: boolean; + panelOpen: boolean | null; } export interface PendingApproval { @@ -2061,8 +2071,10 @@ export function deriveWorkLogEntries( // its private-thinking row suppression on the latter. Only the // derivation-internal collapse key comes off. return enrichGenericThinkingEntries( - collapseDerivedWorkLogEntries(entries).filter(shouldKeepDerivedWorkLogEntry), - ).map(({ collapseKey: _collapseKey, ...entry }) => entry); + collapseBrowserReceipts( + collapseDerivedWorkLogEntries(entries).filter(shouldKeepDerivedWorkLogEntry), + ), + ).map(({ collapseKey: _collapseKey, browserReceipt: _browserReceipt, ...entry }) => entry); } /** The task-notification completion replay re-emits the original Task tool @@ -2206,6 +2218,7 @@ function toDerivedWorkLogEntry( extractRuntimeActivityDetail(activity, payload) ?? extractToolDetail(payload, title ?? activity.summary)); const toolCallId = isTaskActivity ? null : extractToolCallId(payload); + const browserReceipt = deriveBrowserReceipt(payload); const entry: DerivedWorkLogEntry = { id: activity.id, createdAt: activity.createdAt, @@ -2289,6 +2302,9 @@ function toDerivedWorkLogEntry( if (toolCallId) { entry.toolCallId = toolCallId; } + if (browserReceipt) { + entry.browserReceipt = browserReceipt; + } const providerLifecyclePhase = providerLifecyclePhaseFromActivityKind(activity.kind); if (providerLifecyclePhase) { entry.providerLifecyclePhase = providerLifecyclePhase; @@ -2426,6 +2442,80 @@ function collapseDerivedWorkLogEntries( return collapsed; } +/** Turns a run of browser calls on one tab into one durable, readable receipt. */ +function collapseBrowserReceipts( + entries: ReadonlyArray, +): DerivedWorkLogEntry[] { + const collapsed: DerivedWorkLogEntry[] = []; + const activeIndexByTab = new Map(); + + for (const entry of entries) { + const receipt = entry.browserReceipt; + if (!receipt) { + collapsed.push(entry); + continue; + } + const key = [entry.turnId ?? "thread", entry.sourceAgentThreadId ?? "root", receipt.tabId].join( + "\u001f", + ); + const activeIndex = activeIndexByTab.get(key); + if (activeIndex === undefined) { + const receiptEntry = applyBrowserReceiptPresentation(entry, receipt); + collapsed.push(receiptEntry); + if (!receipt.tabClosed) activeIndexByTab.set(key, collapsed.length - 1); + continue; + } + + const previous = collapsed[activeIndex]; + if (!previous) { + collapsed.push(applyBrowserReceiptPresentation(entry, receipt)); + continue; + } + const merged = mergeDerivedWorkLogEntries(previous, entry); + const mergedReceipt = merged.browserReceipt; + if (!mergedReceipt) { + collapsed.push(entry); + continue; + } + collapsed[activeIndex] = applyBrowserReceiptPresentation(merged, mergedReceipt); + if (mergedReceipt.tabClosed) activeIndexByTab.delete(key); + } + + return collapsed; +} + +function applyBrowserReceiptPresentation( + entry: DerivedWorkLogEntry, + receipt: BrowserReceipt, +): DerivedWorkLogEntry { + const parts: string[] = []; + if (receipt.openedUrl) parts.push(`Opened ${truncateInlinePreview(receipt.openedUrl, 72)}`); + if (receipt.finalUrl && receipt.finalUrl !== receipt.openedUrl) { + parts.push(`Final ${truncateInlinePreview(receipt.finalUrl, 72)}`); + } + if (receipt.verifications.length > 0) { + parts.push(`Verified ${formatNaturalList(receipt.verifications)}`); + } + if (receipt.tabClosed) { + parts.push(receipt.panelOpen ? "Closed tab; browser stayed open" : "Closed tab and browser"); + } else { + parts.push("Tab and browser left open"); + } + return { + ...entry, + label: "Browser receipt", + toolTitle: "Browser receipt", + detail: parts.join(" · "), + browserReceipt: receipt, + }; +} + +function formatNaturalList(values: ReadonlyArray): string { + if (values.length <= 1) return values[0] ?? "page"; + if (values.length === 2) return `${values[0]} and ${values[1]}`; + return `${values.slice(0, -1).join(", ")}, and ${values.at(-1)}`; +} + function shouldKeepDerivedWorkLogEntry(entry: DerivedWorkLogEntry): boolean { if (!entry.redactedThinking) { return true; @@ -2653,6 +2743,7 @@ function mergeDerivedWorkLogEntries( const subagentTask = next.subagentTask ?? previous.subagentTask; const spawnedAgentIds = next.spawnedAgentIds ?? previous.spawnedAgentIds; const completedAt = next.completedAt ?? previous.completedAt; + const browserReceipt = mergeBrowserReceipt(previous.browserReceipt, next.browserReceipt); return { ...previous, ...next, @@ -2678,6 +2769,7 @@ function mergeDerivedWorkLogEntries( ...(turnId !== undefined ? { turnId } : {}), ...(subagentTask ? { subagentTask } : {}), ...(spawnedAgentIds ? { spawnedAgentIds } : {}), + ...(browserReceipt ? { browserReceipt } : {}), }; } @@ -3456,6 +3548,7 @@ interface ToolCallIdentity { readonly serverOrNamespace: string | null; readonly tool: string | null; readonly input: unknown; + readonly result: unknown; } interface SemanticToolPresentation { @@ -3486,6 +3579,12 @@ function extractToolCallIdentity(payload: Record | null): ToolC : data && Object.prototype.hasOwnProperty.call(data, "arguments") ? data.arguments : undefined; + const result = + item && Object.prototype.hasOwnProperty.call(item, "result") + ? item.result + : data && Object.prototype.hasOwnProperty.call(data, "result") + ? data.result + : payload?.result; if (!serverOrNamespace && !tool) { return null; @@ -3496,6 +3595,92 @@ function extractToolCallIdentity(payload: Record | null): ToolC serverOrNamespace, tool, input, + result, + }; +} + +function deriveBrowserReceipt(payload: Record | null): BrowserReceipt | null { + const identity = extractToolCallIdentity(payload); + const tool = identity?.tool?.trim().toLowerCase() ?? ""; + if ( + !identity || + !toolNamespaceMatches(identity.serverOrNamespace, "browser", "threadlines_browser") || + !tool.startsWith("browser_") + ) { + return null; + } + + const result = structuredToolResult(identity.result); + const closedTab = asRecord(result?.closedTab ?? result?.closed_tab); + const tabId = + asTrimmedString(result?.tabId ?? result?.tab_id) ?? + asTrimmedString(closedTab?.id) ?? + argumentValue(identity.input, "tabId", "tab_id"); + if (!tabId) return null; + + const inputUrl = argumentValue(identity.input, "url", "href"); + const finalUrl = + asTrimmedString(closedTab?.url) ?? asTrimmedString(result?.url) ?? inputUrl ?? null; + const verifications: string[] = []; + if (tool === "browser_open_tab" || tool === "browser_navigate") { + verifications.push("address", "load state"); + } + if (tool === "browser_snapshot") verifications.push("page content"); + if (tool === "browser_screenshot") verifications.push("appearance"); + if (tool === "browser_status") verifications.push("address", "load state"); + if (tool === "browser_wait_for") { + const input = asRecord(identity.input); + if (asTrimmedString(input?.urlContains ?? input?.url_contains)) { + verifications.push("address"); + } + if (asTrimmedString(input?.text) || input?.target !== undefined) { + verifications.push("page content"); + } + if (verifications.length === 0) verifications.push("page state"); + } + + return { + tabId, + openedUrl: inputUrl ?? finalUrl, + finalUrl, + verifications, + tabClosed: tool === "browser_close_tab" && closedTab !== null, + panelOpen: typeof result?.panelOpen === "boolean" ? result.panelOpen : null, + }; +} + +function structuredToolResult(value: unknown): Record | null { + const result = asRecord(value) ?? parseJsonRecord(value); + if (!result) return null; + return ( + asRecord(result.structuredContent ?? result.structured_content) ?? + parseJsonRecord(result.structuredContent ?? result.structured_content) ?? + result + ); +} + +function parseJsonRecord(value: unknown): Record | null { + if (typeof value !== "string" || !value.trim().startsWith("{")) return null; + try { + return asRecord(JSON.parse(value)); + } catch { + return null; + } +} + +function mergeBrowserReceipt( + previous: BrowserReceipt | undefined, + next: BrowserReceipt | undefined, +): BrowserReceipt | undefined { + if (!previous) return next; + if (!next || previous.tabId !== next.tabId) return previous; + return { + tabId: previous.tabId, + openedUrl: previous.openedUrl ?? next.openedUrl, + finalUrl: next.finalUrl ?? previous.finalUrl, + verifications: uniqueStrings([...previous.verifications, ...next.verifications]), + tabClosed: previous.tabClosed || next.tabClosed, + panelOpen: next.panelOpen ?? previous.panelOpen, }; } @@ -3644,9 +3829,42 @@ function browserAutomationDetail(identity: ToolCallIdentity): string { const url = directUrl ?? codeUrl; const tool = identity.tool?.trim().toLowerCase(); - if (tool && /^(?:open|goto|navigate|new_tab)$/u.test(tool)) { + if (tool && /^(?:open|goto|new_tab|browser_open_tab|open_tab)$/u.test(tool)) { return url ? `Opening ${truncateInlinePreview(url, 72)}` : "Opening browser page"; } + if (tool && /^(?:browser_navigate|navigate)$/u.test(tool)) { + return url ? `Navigating to ${truncateInlinePreview(url, 72)}` : "Navigating browser page"; + } + if (tool && /^(?:browser_close_tab|close_tab)$/u.test(tool)) { + const tabId = argumentValue(identity.input, "tabId", "tab_id"); + return tabId ? `Closed browser tab ${truncateInlinePreview(tabId, 32)}` : "Closed browser tab"; + } + if (tool && /^(?:browser_select_tab|select_tab)$/u.test(tool)) { + const tabId = argumentValue(identity.input, "tabId", "tab_id"); + return tabId + ? `Selected browser tab ${truncateInlinePreview(tabId, 32)}` + : "Selected browser tab"; + } + if (tool && /^(?:browser_tabs|tabs)$/u.test(tool)) { + return "Listed browser tabs"; + } + if (tool && /^(?:browser_snapshot|snapshot)$/u.test(tool)) { + return "Verified browser page"; + } + if (tool && /^(?:browser_wait_for|wait_for)$/u.test(tool)) { + return "Waited for browser page"; + } + if (tool && /^(?:browser_)?screenshot$/u.test(tool)) { + return "Captured browser screenshot"; + } + if ( + tool && + /^(?:browser_)?(?:click|type|press|scroll|move|drag|evaluate|resize|set_appearance)$/u.test( + tool, + ) + ) { + return "Interacted with browser page"; + } if (code && /\.goto\(/u.test(code) && url) { return `Opening ${truncateInlinePreview(url, 72)}`; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index b2cb137a5..b6815c627 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -432,6 +432,12 @@ export const DesktopPreviewNavigationBlockedSchema = Schema.Struct({ }); export type DesktopPreviewNavigationBlocked = typeof DesktopPreviewNavigationBlockedSchema.Type; +/** Human input reached a guest page, so agent-owned cleanup must yield to the user. */ +export const DesktopPreviewUserControlSchema = Schema.Struct({ + webContentsId: Schema.Number, +}); +export type DesktopPreviewUserControl = typeof DesktopPreviewUserControlSchema.Type; + export const DesktopPreviewEvaluateInputSchema = Schema.Struct({ webContentsId: Schema.Number, expression: Schema.String, @@ -986,6 +992,8 @@ export interface DesktopBridge { onPreviewNavigationBlocked?: ( listener: (event: DesktopPreviewNavigationBlocked) => void, ) => () => void; + /** Fires for genuine keyboard, pointer, or wheel input inside a guest page. */ + onPreviewUserControl?: (listener: (event: DesktopPreviewUserControl) => void) => () => void; previewClearBrowsingData?: () => Promise; previewClearCache?: () => Promise; setTheme: (theme: DesktopTheme) => Promise; diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index c8d8279f7..9fa26161d 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -261,6 +261,16 @@ export const PreviewAutomationTabSchema = Schema.Struct({ export const PreviewAutomationTabsSchema = Schema.Struct({ tabs: Schema.Array(PreviewAutomationTabSchema), + /** Whether the browser panel remains visible after this operation. */ + panelOpen: Schema.Boolean, + /** The page that was closed, present only on a successful close. */ + closedTab: Schema.optionalKey( + Schema.Struct({ + id: Schema.String, + title: Schema.String, + url: Schema.String, + }), + ), }); /** Pin the agent to another tab, optionally leaving the user's view alone. */