From c307fc48058599e159f971439d6d336a254ffaf3 Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:47:49 -0400 Subject: [PATCH] fix(web): agent page resizes now scale to fit the browser panel When an agent called browser_resize, the new size reached the page but not the tab. The frame kept filling the panel while the page rendered at the wider viewport, so it came out cropped. The result also reported the panel's size instead of the page's, so the agent read its resize as having done nothing and resized again. An agent resize is now recorded on the tab the same way a drag on the frame's edges is: the panel scales the page down to fit, keeps its aspect ratio, shows the size in the device toolbar, and leaves the chat split alone. The status reports the page's CSS viewport, the activity line says "Agent resized 1280x800", and the tool description tells the agent the panel scales rather than crops. --- apps/server/src/mcp/browserTools.ts | 2 +- .../browser/PreviewAutomationMount.tsx | 18 +++++++ .../browser/previewAutomationHost.test.ts | 50 +++++++++++++++++++ .../browser/previewAutomationHost.ts | 28 +++++++++-- 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/apps/server/src/mcp/browserTools.ts b/apps/server/src/mcp/browserTools.ts index 1750edf4..20a5edcb 100644 --- a/apps/server/src/mcp/browserTools.ts +++ b/apps/server/src/mcp/browserTools.ts @@ -230,7 +230,7 @@ export const BrowserEvaluateTool = changesThePage( export const BrowserResizeTool = changesThePage( Tool.make("browser_resize", { description: - "Resize the page the user is looking at, to check a layout at a given width. Pass null for both to let it fill the panel and reflow with it. Returns the page state at the new size.", + "Set the page's CSS viewport, to check a layout at a given width. The panel scales the page down to fit beside the chat, so a desktop width stays whole on screen rather than cropped, and the user's window does not change. Pass null for both to let the page fill the panel and reflow with it. Returns the page state at the new size.", parameters: PreviewAutomationResizeInputSchema, success: PreviewAutomationStatusSchema, failure: PreviewAutomationErrorSchema, diff --git a/apps/web/src/components/browser/PreviewAutomationMount.tsx b/apps/web/src/components/browser/PreviewAutomationMount.tsx index fac45c04..d9acd066 100644 --- a/apps/web/src/components/browser/PreviewAutomationMount.tsx +++ b/apps/web/src/components/browser/PreviewAutomationMount.tsx @@ -75,6 +75,7 @@ export function PreviewAutomationMount({ const setAgentActivity = useBrowserPanelStore((store) => store.setAgentActivity); const selectTab = useBrowserPanelStore((store) => store.selectTab); const setTabUrl = useBrowserPanelStore((store) => store.setTabUrl); + const setTabViewport = useBrowserPanelStore((store) => store.setTabViewport); const setPendingBrowserApproval = useBrowserPanelStore( (store) => store.setPendingBrowserApproval, ); @@ -290,9 +291,25 @@ export function PreviewAutomationMount({ })); }, viewport: () => { + // The size the page was given, when it was given one. The frame is + // scaled to fit the panel, so its on-screen box is not the page's own + // viewport; a responsive page is exactly as big as its element. Read + // now rather than from the state above: a resize in this same request + // has just written it. + const current = selectThreadBrowserState( + useBrowserPanelStore.getState().browserStateByThreadKey, + threadRef, + ); + const fixed = current.tabs.find((entry) => entry.id === tabId)?.viewport; + if (fixed !== undefined && fixed.width !== null && fixed.height !== null) { + return { width: fixed.width, height: fixed.height }; + } const rect = webview?.getBoundingClientRect(); return { width: Math.round(rect?.width ?? 0), height: Math.round(rect?.height ?? 0) }; }, + setViewport: (viewport) => { + if (tabId !== "") setTabViewport(threadRef, tabId, viewport); + }, // The address belongs to the element, so this is the one operation the // main process cannot do on the agent's behalf. navigate: async (url) => { @@ -343,6 +360,7 @@ export function PreviewAutomationMount({ setAgentTab, setPendingBrowserApproval, setTabUrl, + setTabViewport, threadRef, ], ); diff --git a/apps/web/src/components/browser/previewAutomationHost.test.ts b/apps/web/src/components/browser/previewAutomationHost.test.ts index 7d06db81..f72ed973 100644 --- a/apps/web/src/components/browser/previewAutomationHost.test.ts +++ b/apps/web/src/components/browser/previewAutomationHost.test.ts @@ -17,6 +17,7 @@ const handlerFor = ( webContentsId, navigate, viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, tabs: () => [], selectTab: () => {}, @@ -73,6 +74,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, tabs: () => [], selectTab: () => {}, @@ -130,6 +132,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: (point) => points.push(point), tabs: () => [], selectTab: () => {}, @@ -156,6 +159,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: (point) => points.push(point), tabs: () => [], selectTab: () => {}, @@ -183,6 +187,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, onAgentActivity: (activity) => seen.push(activity), tabs: () => [], @@ -197,6 +202,47 @@ describe("createPreviewAutomationHandler", () => { expect(seen[0]?.detail).toBe("120, 60 \u2192 300, 60"); }); + it("records an agent resize on the tab and answers with the size it asked for", async () => { + // Told to the guest alone, the page rendered at the new width inside a + // frame that still filled the panel, so it came out cropped -- and the + // answer reported the panel's size, so the agent read its resize as having + // done nothing and asked again. + const applied: unknown[] = []; + const seen: AgentActivity[] = []; + let fixed: { width: number | null; height: number | null } = { width: null, height: null }; + const handle = createPreviewAutomationHandler( + { + previewSetViewport: (input: unknown) => { + applied.push(input); + return Promise.resolve(); + }, + previewStatus: () => Promise.resolve({ url: "http://x/", title: "X", loading: false }), + } as unknown as DesktopBridge, + () => ({ + webContentsId: 42, + navigate: () => Promise.resolve(), + viewport: () => + fixed.width !== null && fixed.height !== null + ? { width: fixed.width, height: fixed.height } + : { width: 800, height: 600 }, + setViewport: (viewport) => { + fixed = viewport; + }, + onAgentPoint: () => {}, + onAgentActivity: (activity) => seen.push(activity), + tabs: () => [], + selectTab: () => {}, + }), + ); + + const response = await handle(request("resize", { width: 1280, height: 800 })); + + expect(applied).toEqual([{ webContentsId: 42, width: 1280, height: 800 }]); + expect(fixed).toEqual({ width: 1280, height: 800 }); + expect(response.result).toMatchObject({ width: 1280, height: 800 }); + expect(seen.at(-1)?.detail).toBe("1280\u00d7800"); + }); + it("reports every tab, marking the user's and its own", async () => { // The agent pins itself to a tab. Without a way to see the others it // reports on the one it is pinned to and states that nothing else is open, @@ -209,6 +255,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, onAgentActivity: () => {}, selectTab: () => {}, @@ -270,6 +317,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, onAgentActivity: () => {}, selectTab: () => {}, @@ -322,6 +370,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, tabs: () => [], selectTab: () => {}, @@ -349,6 +398,7 @@ describe("createPreviewAutomationHandler", () => { webContentsId: 42, navigate: () => Promise.resolve(), viewport: () => ({ width: 800, height: 600 }), + setViewport: () => {}, onAgentPoint: () => {}, tabs: () => [], selectTab: () => {}, diff --git a/apps/web/src/components/browser/previewAutomationHost.ts b/apps/web/src/components/browser/previewAutomationHost.ts index e5a675c4..67533d80 100644 --- a/apps/web/src/components/browser/previewAutomationHost.ts +++ b/apps/web/src/components/browser/previewAutomationHost.ts @@ -62,10 +62,18 @@ export interface PreviewAutomationHostTarget { * Everything else is a CDP command and goes over the bridge. */ readonly navigate: (url: string) => Promise; - /** How big the page is right now. The panel is the only one that knows: the - * main process cannot see the element and this module should not reach for - * it. A question about layout is a question about this. */ + /** How big the page is right now, in its own CSS pixels: the size it was + * given, or the panel's when it fills it. The panel is the only one that + * knows: the main process cannot see the element and this module should not + * reach for it. A question about layout is a question about this. */ readonly viewport: () => { width: number; height: number }; + /** + * Records the CSS viewport the agent asked for on the tab, the same as a drag + * on the frame's edges: the panel scales the frame to fit and the device + * toolbar shows the size. The guest is told separately, by this module, so + * the answer read straight afterwards does not race a render. + */ + readonly setViewport: (viewport: { width: number | null; height: number | null }) => void; /** Where the agent just acted, so the panel can show it happening. */ readonly onAgentPoint: (point: { x: number; @@ -169,6 +177,13 @@ function describeSubject(operation: PreviewAutomationOperation, input: unknown): if (operation === "press") { return typeof value.key === "string" ? value.key : null; } + if (operation === "resize") { + // The size is the whole story of a resize; "resized" alone reads as a + // panel that changed shape for no reason. + return typeof value.width === "number" && typeof value.height === "number" + ? `${Math.round(value.width)}×${Math.round(value.height)}` + : "to fit the panel"; + } if (operation === "selectTab") { if (typeof value.tabId === "string") return value.tabId; return typeof value.index === "number" ? `tab ${value.index + 1}` : null; @@ -450,9 +465,14 @@ async function dispatch( height: shot.height, }; } - case "resize": + case "resize": { await call(bridge.previewSetViewport, input); + // Recorded on the tab only once the guest has taken it, so a frame never + // shows a size the page was not given. + const size = input as { width?: number | null; height?: number | null }; + target.setViewport({ width: size.width ?? null, height: size.height ?? null }); return toStatus(target.tabId ?? "", await call(bridge.previewStatus, {}), target.viewport()); + } case "setAppearance": await call(bridge.previewSetColorScheme, input); return toStatus(target.tabId ?? "", await call(bridge.previewStatus, {}), target.viewport());