From 62918e68fa4ec8fc0c9607e7eea8288bfb2316e0 Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Wed, 1 Jul 2026 15:41:18 -0700 Subject: [PATCH] fix(panel): live-update when unfocused + non-distorting responsive resize (v0.6.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two field-reported issues. 1. "Only live-navigates when the view is focused." The screencast stalled unless the panel had focus. Fixed defensively across the plausible causes: - Pin the page focused/visible over CDP (Emulation.setFocusEmulationEnabled) so it never throttles rendering when the operator's console view isn't the focused one. - Headed windows launch with anti-backgrounding flags (--disable-renderer-backgrounding, --disable-backgrounding-occluded-windows, --disable-background-timer-throttling) so an occluded window keeps drawing. - The panel reconnects (if the socket dropped) / forces a fresh frame on visibilitychange + focus, so switching back snaps to current instantly. 2. "Responsive is buggy." Root cause: the canvas stretched the old-aspect frame while the viewport round-tripped, so it distorted; and a drag re-armed the screencast on every observation. - canvas object-fit: contain — letterbox during the catch-up instead of distorting. - letterbox-aware input coordinate mapping. - dedupe set_viewport server-side (skip unchanged size) + debounce 180→220ms. No stream regression: nav still paints every page, resize still reshapes to the dock (validated live). 47 host-free tests pass; ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 12 ++++++++++++ README.md | 8 +++++--- browser_panel.py | 17 ++++++++++++++--- browser_stream.py | 17 +++++++++++++++-- protoagent.plugin.yaml | 2 +- pyproject.toml | 2 +- runtime.py | 16 ++++++++++++---- tests/test_agent_browser.py | 7 +++++-- tests/test_runtime.py | 17 +++++++++++++++-- 9 files changed, 80 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab73ef..6e277fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## v0.6.2 +- **Keeps live-updating when the panel isn't focused.** The screencast used to stall unless the + panel had focus. Fixed on three fronts: the page is now pinned focused/visible over CDP + (`Emulation.setFocusEmulationEnabled`) so it never throttles rendering; headed windows launch + with anti-backgrounding flags (`--disable-renderer-backgrounding` &co) so an occluded window + keeps drawing; and the panel reconnects / forces a fresh frame on `visibilitychange` + `focus` + so switching back snaps to current instantly. +- **Responsive resize no longer distorts or thrashes.** The canvas uses `object-fit: contain` + (no more stretched/squished frames while the viewport catches up), input mapping is + letterbox-aware, and identical resize observations are deduped server-side so a drag doesn't + re-arm the screencast repeatedly. Debounce nudged to 220ms. + ## v0.6.1 - **Full-stretch, responsive viewport.** The panel now resizes Chrome's layout viewport to your dock's size (× device-pixel-ratio) over CDP as you resize/expand/collapse it — the page diff --git a/README.md b/README.md index baddbd6..6ce9893 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,11 @@ point. The viewport is **full-stretch and responsive** — it resizes Chrome's layout viewport to your dock's size (× device-pixel-ratio) as you expand/collapse it, so the page reflows to fill rather - than sitting as a fixed box, and the screencast **re-arms on every navigation** so you see the - agent move through pages and sub-pages (not just the first load). Tune sharpness with - `stream_quality`. + than sitting as a fixed box (letterboxed with `object-fit` during the resize so it never + distorts). The screencast **re-arms on every navigation** so you see the agent move through + pages and sub-pages (not just the first load), and it **keeps updating even when the panel isn't + focused** (the page is pinned focused/visible over CDP; headed windows launch with + anti-backgrounding flags). Tune sharpness with `stream_quality`. When no page is open the panel shows a **Start button** (not a dead end). Set `home_url` to a page and the panel **auto-opens** it — a homepage — otherwise Start opens `about:blank`. diff --git a/browser_panel.py b/browser_panel.py index d22c769..22c7a2f 100644 --- a/browser_panel.py +++ b/browser_panel.py @@ -165,7 +165,7 @@ async def _nav(body: dict = Body(...)): background:var(--pl-color-fg-muted,#9aa0aa)} .dot.ok{background:#22c55e}.dot.err{background:#ef4444} .stage{height:calc(100% - 38px);position:relative;background:var(--pl-color-bg-inset);overflow:hidden} - canvas{width:100%;height:100%;display:block;outline:none;touch-action:none} + canvas{width:100%;height:100%;display:block;outline:none;touch-action:none;object-fit:contain} #msg{display:none;position:absolute;inset:0;align-items:center;justify-content:center; padding:24px;box-sizing:border-box} .card{max-width:460px;text-align:center} @@ -261,8 +261,13 @@ async def _nav(body: dict = Body(...)): // ── input forwarding: canvas events → CDP Input.dispatch* (coords in CSS px) ──── function send(o){ if(ws&&ws.readyState===1) ws.send(JSON.stringify(o)); } function mods(e){ return {shift:e.shiftKey,ctrl:e.ctrlKey,alt:e.altKey,meta:e.metaKey}; } +// map a client point → page CSS px, accounting for object-fit:contain letterboxing +// (the frame's aspect may differ from the canvas box for a moment after a resize). function pos(e){ const r=cv.getBoundingClientRect(); - return { x:(e.clientX-r.left)/r.width*devW, y:(e.clientY-r.top)/r.height*devH }; } + const bw=cv.width||devW, bh=cv.height||devH; // backing store = frame pixels + const s=Math.min(r.width/bw, r.height/bh); // contain scale + const dw=bw*s, dh=bh*s, ox=(r.width-dw)/2, oy=(r.height-dh)/2; + return { x:(e.clientX-r.left-ox)/dw*devW, y:(e.clientY-r.top-oy)/dh*devH }; } const BTN={0:"left",1:"middle",2:"right"}; cv.addEventListener("mousedown",(e)=>{ e.preventDefault(); cv.focus(); const p=pos(e); send({t:"mouse",action:"down",x:p.x,y:p.y,button:BTN[e.button]||"left",clickCount:e.detail||1,buttons:e.buttons,...mods(e)}); }); @@ -288,7 +293,13 @@ async def _nav(body: dict = Body(...)): function panelSize(){ const r=cv.parentElement.getBoundingClientRect(); return { w:Math.round(r.width), h:Math.round(r.height), dpr:Math.min(window.devicePixelRatio||1, 2) }; } function sendResize(){ const s=panelSize(); if(s.w>10 && s.h>10) send({t:"resize",w:s.w,h:s.h,dpr:s.dpr}); } -new ResizeObserver(()=>{ clearTimeout(rzTimer); rzTimer=setTimeout(sendResize, 180); }).observe(cv.parentElement); +new ResizeObserver(()=>{ clearTimeout(rzTimer); rzTimer=setTimeout(sendResize, 220); }).observe(cv.parentElement); + +// When the panel becomes visible/focused again, the console may have throttled it while +// hidden — reconnect if the socket dropped, else force a fresh frame so it snaps to current. +function onVisible(){ if(document.hidden) return; if(!connected) connect(); else { send({t:"refresh"}); sendResize(); } } +document.addEventListener("visibilitychange", onVisible); +window.addEventListener("focus", onVisible); // ── boot ONCE — on the handshake (so apiFetch has the bearer for the gated ticket) // or an 800ms fallback for a standalone/older host that posts no init. ────────── diff --git a/browser_stream.py b/browser_stream.py index 0faa6b9..627335c 100644 --- a/browser_stream.py +++ b/browser_stream.py @@ -218,6 +218,7 @@ def __init__(self, page_ws_url: str, frame_cb, quality: int = 80): self._frame_cb = frame_cb self._quality = max(1, min(int(quality or 80), 100)) self._cast = (1280, 800) # current screencast max frame size (device px) + self._last_vp = None # last applied (css_w, css_h, scale) — dedupe resize thrash self._ws = None self._id = 0 self._last_arm = 0.0 # debounce re-arms (multi-frame pages fire many events) @@ -250,22 +251,34 @@ async def _arm_cast(self): async def start_screencast(self, max_w: int = 1280, max_h: int = 800): await self._send("Page.enable") # also enables frameNavigated / loadEventFired for re-arm + # Treat the page as permanently focused + visible. Otherwise Chrome throttles rendering + # when the page isn't the focused surface, and the screencast stalls until the operator + # clicks back into the panel ("only live-navigates when focused"). + await self._send("Emulation.setFocusEmulationEnabled", {"enabled": True}) self._cast = (max_w, max_h) await self._arm_cast() async def set_viewport(self, w, h, dpr=1.0): """Resize Chrome's layout viewport to the panel and re-arm the screencast at the - matching frame size — so the page reflows to fill, and stays crisp.""" + matching frame size — so the page reflows to fill, and stays crisp. Deduped: an + unchanged size is a no-op (a drag fires many identical observations).""" cw, ch, scale, mw, mh = viewport_metrics(w, h, dpr) + if (cw, ch, scale) == self._last_vp: + return + self._last_vp = (cw, ch, scale) await self._send("Emulation.setDeviceMetricsOverride", {"width": cw, "height": ch, "deviceScaleFactor": scale, "mobile": False}) self._cast = (mw, mh) await self._arm_cast() async def dispatch(self, msg: dict): - if msg.get("t") == "resize": + t = msg.get("t") + if t == "resize": await self.set_viewport(msg.get("w"), msg.get("h"), msg.get("dpr", 1)) return + if t == "refresh": # panel became visible again → force a fresh frame + await self._arm_cast() + return cmd = input_to_cdp(msg) if cmd: await self._send(*cmd) diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 137e54a..752e7bb 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: agent_browser name: Agent Browser -version: 0.6.1 +version: 0.6.2 description: >- Browser automation for protoAgent, backed by **agent-browser** (vercel-labs) — a fast native-Rust CLI/daemon that drives Chrome over CDP with accessibility-tree diff --git a/pyproject.toml b/pyproject.toml index 97b4e37..d7c585b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agent-browser-plugin" -version = "0.6.1" +version = "0.6.2" description = "Browser-automation plugin for protoAgent, backed by vercel-labs/agent-browser (tools + skill + workflows + an interactive, drivable browser panel)." requires-python = ">=3.11" diff --git a/runtime.py b/runtime.py index 70b3b4a..a56e2b2 100644 --- a/runtime.py +++ b/runtime.py @@ -23,8 +23,18 @@ def launch_flags(cfg: dict | None) -> list[str]: cfg = cfg or {} f: list[str] = [] headed = bool(cfg.get("headed")) + # Extra Chrome launch args (comma/newline separated); anti-detection + anti-throttle + # flags get merged in below. + args = [a.strip() for a in str(cfg.get("browser_args") or "").replace("\n", ",").split(",") if a.strip()] if headed: f.append("--headed") + # A headed window gets throttled/paused when it loses focus or is occluded, which + # stalls the live screencast. Keep it rendering so the panel updates even when the + # operator is looking at another window. + for a in ("--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding", + "--disable-background-timer-throttling"): + if a not in args: + args.append(a) if str(cfg.get("profile") or "").strip(): f += ["--profile", str(cfg["profile"]).strip()] if str(cfg.get("device") or "").strip(): @@ -37,10 +47,8 @@ def launch_flags(cfg: dict | None) -> list[str]: f += ["--max-output", str(int(cfg["max_output"]))] # ── anti-detection ────────────────────────────────────────────────────────── - # Extra Chrome launch args (comma/newline separated) + a UA override. `stealth` layers - # on the common evasions: drop the `navigator.webdriver` automation flag, and (when - # headless, where the UA says "HeadlessChrome") swap in a real desktop UA. - args = [a.strip() for a in str(cfg.get("browser_args") or "").replace("\n", ",").split(",") if a.strip()] + # `stealth` layers on the common evasions: drop the `navigator.webdriver` automation + # flag, and (when headless, where the UA says "HeadlessChrome") swap in a real desktop UA. ua = str(cfg.get("user_agent") or "").strip() if bool(cfg.get("stealth")): if "--disable-blink-features=AutomationControlled" not in args: diff --git a/tests/test_agent_browser.py b/tests/test_agent_browser.py index e9b4b4b..9fdf242 100644 --- a/tests/test_agent_browser.py +++ b/tests/test_agent_browser.py @@ -26,10 +26,11 @@ def _toolmap(cfg=None): async def test_open_passes_url_and_curated_launch_flags(monkeypatch): rec = [] monkeypatch.setattr(tools.subprocess, "run", fake_run(stdout="OPENED", record=rec)) - t = _toolmap({"binary": "ab", "headed": True, "allowed_domains": "x.com", "max_output": 500}) + # headless so argv is clean (headed injects anti-throttle --args; covered in test_runtime) + t = _toolmap({"binary": "ab", "allowed_domains": "x.com", "max_output": 500}) out = await t["browser_open"].ainvoke({"url": "https://x.com"}) assert "OPENED" in out - assert rec[-1] == ["ab", "--headed", "--allowed-domains", "x.com", "--max-output", "500", "open", "https://x.com"] + assert rec[-1] == ["ab", "--allowed-domains", "x.com", "--max-output", "500", "open", "https://x.com"] async def test_open_blank_url_omits_it(monkeypatch): @@ -159,6 +160,8 @@ def test_panel_page_wires_canvas_stream_and_input(): assert 'u.protocol==="https:" ? "wss:" : "ws:"' in html # http→ws upgrade assert 'send({t:"mouse"' in html and 'send({t:"key"' in html # input forwarding assert "ResizeObserver" in html and 'send({t:"resize"' in html # responsive viewport tracking + assert "object-fit:contain" in html # no distortion during resize + assert "visibilitychange" in html and 'send({t:"refresh"' in html # refresh when re-shown assert "/api/plugins/agent_browser/nav" in html and "kit.apiFetch" in html # nav via gated route assert "startBrowser" in html and 'const HOME="";' in html # empty-state Start; blank home default # the removed dashboard-embed / screenshot modes leave no trace: diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 5d3d962..dbea25f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -12,12 +12,25 @@ def test_default_is_empty(): def test_curated_flags_stable_order(): - f = rt.launch_flags({"headed": True, "profile": "P", "device": "iPhone 16 Pro", + # headless keeps the argv clean (headed injects anti-throttle --args, tested separately) + f = rt.launch_flags({"profile": "P", "device": "iPhone 16 Pro", "allowed_domains": "x.com", "confirm_actions": "nav", "max_output": 500}) - assert f == ["--headed", "--profile", "P", "--device", "iPhone 16 Pro", + assert f == ["--profile", "P", "--device", "iPhone 16 Pro", "--allowed-domains", "x.com", "--confirm-actions", "nav", "--max-output", "500"] +def test_headed_injects_anti_throttle_args(): + f = rt.launch_flags({"headed": True}) + assert f[0] == "--headed" and f[1] == "--args" + aset = set(f[2].split(",")) + assert {"--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding", + "--disable-background-timer-throttling"} <= aset # keep a headed window rendering unfocused + + +def test_headless_stays_clean(): + assert rt.launch_flags({"allowed_domains": "x.com"}) == ["--allowed-domains", "x.com"] # no --args + + def test_stealth_headless_adds_automation_arg_and_real_ua(): f = rt.launch_flags({"stealth": True}) # headless by default assert f[f.index("--user-agent") + 1].startswith("Mozilla/5.0")