fix(panel): live-update when unfocused + non-distorting responsive resize (v0.6.2)

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v0.6.2
+- **Keeps live-updating when the panel isn't focused.** The screencast used to stall unless the
+  panel had focus. Fixed on three fronts: the page is now pinned focused/visible over CDP
+  (`Emulation.setFocusEmulationEnabled`) so it never throttles rendering; headed windows launch
+  with anti-backgrounding flags (`--disable-renderer-backgrounding` &co) so an occluded window
+  keeps drawing; and the panel reconnects / forces a fresh frame on `visibilitychange` + `focus`
+  so switching back snaps to current instantly.
+- **Responsive resize no longer distorts or thrashes.** The canvas uses `object-fit: contain`
+  (no more stretched/squished frames while the viewport catches up), input mapping is
+  letterbox-aware, and identical resize observations are deduped server-side so a drag doesn't
+  re-arm the screencast repeatedly. Debounce nudged to 220ms.
+
 ## v0.6.1
 - **Full-stretch, responsive viewport.** The panel now resizes Chrome's layout viewport to
   your dock's size (× device-pixel-ratio) over CDP as you resize/expand/collapse it — the page

README.md

@@ -47,9 +47,11 @@ point.
 
   The viewport is **full-stretch and responsive** — it resizes Chrome's layout viewport to your
   dock's size (× device-pixel-ratio) as you expand/collapse it, so the page reflows to fill rather
-  than sitting as a fixed box, and the screencast **re-arms on every navigation** so you see the
-  agent move through pages and sub-pages (not just the first load). Tune sharpness with
-  `stream_quality`.
+  than sitting as a fixed box (letterboxed with `object-fit` during the resize so it never
+  distorts). The screencast **re-arms on every navigation** so you see the agent move through
+  pages and sub-pages (not just the first load), and it **keeps updating even when the panel isn't
+  focused** (the page is pinned focused/visible over CDP; headed windows launch with
+  anti-backgrounding flags). Tune sharpness with `stream_quality`.
 
   When no page is open the panel shows a **Start button** (not a dead end). Set `home_url` to a
   page and the panel **auto-opens** it — a homepage — otherwise Start opens `about:blank`.

browser_panel.py

@@ -165,7 +165,7 @@ async def _nav(body: dict = Body(...)):
     background:var(--pl-color-fg-muted,#9aa0aa)}
   .dot.ok{background:#22c55e}.dot.err{background:#ef4444}
   .stage{height:calc(100% - 38px);position:relative;background:var(--pl-color-bg-inset);overflow:hidden}
-  canvas{width:100%;height:100%;display:block;outline:none;touch-action:none}
+  canvas{width:100%;height:100%;display:block;outline:none;touch-action:none;object-fit:contain}
   #msg{display:none;position:absolute;inset:0;align-items:center;justify-content:center;
     padding:24px;box-sizing:border-box}
   .card{max-width:460px;text-align:center}
@@ -261,8 +261,13 @@ async def _nav(body: dict = Body(...)):
 // ── input forwarding: canvas events → CDP Input.dispatch* (coords in CSS px) ────
 function send(o){ if(ws&&ws.readyState===1) ws.send(JSON.stringify(o)); }
 function mods(e){ return {shift:e.shiftKey,ctrl:e.ctrlKey,alt:e.altKey,meta:e.metaKey}; }
+// map a client point → page CSS px, accounting for object-fit:contain letterboxing
+// (the frame's aspect may differ from the canvas box for a moment after a resize).
 function pos(e){ const r=cv.getBoundingClientRect();
-  return { x:(e.clientX-r.left)/r.width*devW, y:(e.clientY-r.top)/r.height*devH }; }
+  const bw=cv.width||devW, bh=cv.height||devH;              // backing store = frame pixels
+  const s=Math.min(r.width/bw, r.height/bh);                // contain scale
+  const dw=bw*s, dh=bh*s, ox=(r.width-dw)/2, oy=(r.height-dh)/2;
+  return { x:(e.clientX-r.left-ox)/dw*devW, y:(e.clientY-r.top-oy)/dh*devH }; }
 const BTN={0:"left",1:"middle",2:"right"};
 cv.addEventListener("mousedown",(e)=>{ e.preventDefault(); cv.focus(); const p=pos(e);
   send({t:"mouse",action:"down",x:p.x,y:p.y,button:BTN[e.button]||"left",clickCount:e.detail||1,buttons:e.buttons,...mods(e)}); });
@@ -288,7 +293,13 @@ async def _nav(body: dict = Body(...)):
 function panelSize(){ const r=cv.parentElement.getBoundingClientRect();
   return { w:Math.round(r.width), h:Math.round(r.height), dpr:Math.min(window.devicePixelRatio||1, 2) }; }
 function sendResize(){ const s=panelSize(); if(s.w>10 && s.h>10) send({t:"resize",w:s.w,h:s.h,dpr:s.dpr}); }
-new ResizeObserver(()=>{ clearTimeout(rzTimer); rzTimer=setTimeout(sendResize, 180); }).observe(cv.parentElement);
+new ResizeObserver(()=>{ clearTimeout(rzTimer); rzTimer=setTimeout(sendResize, 220); }).observe(cv.parentElement);
+
+// When the panel becomes visible/focused again, the console may have throttled it while
+// hidden — reconnect if the socket dropped, else force a fresh frame so it snaps to current.
+function onVisible(){ if(document.hidden) return; if(!connected) connect(); else { send({t:"refresh"}); sendResize(); } }
+document.addEventListener("visibilitychange", onVisible);
+window.addEventListener("focus", onVisible);
 
 // ── boot ONCE — on the handshake (so apiFetch has the bearer for the gated ticket)
 // or an 800ms fallback for a standalone/older host that posts no init. ──────────

browser_stream.py

@@ -218,6 +218,7 @@ def __init__(self, page_ws_url: str, frame_cb, quality: int = 80):
         self._frame_cb = frame_cb
         self._quality = max(1, min(int(quality or 80), 100))
         self._cast = (1280, 800)      # current screencast max frame size (device px)
+        self._last_vp = None          # last applied (css_w, css_h, scale) — dedupe resize thrash
         self._ws = None
         self._id = 0
         self._last_arm = 0.0          # debounce re-arms (multi-frame pages fire many events)
@@ -250,22 +251,34 @@ async def _arm_cast(self):
 
     async def start_screencast(self, max_w: int = 1280, max_h: int = 800):
         await self._send("Page.enable")   # also enables frameNavigated / loadEventFired for re-arm
+        # Treat the page as permanently focused + visible. Otherwise Chrome throttles rendering
+        # when the page isn't the focused surface, and the screencast stalls until the operator
+        # clicks back into the panel ("only live-navigates when focused").
+        await self._send("Emulation.setFocusEmulationEnabled", {"enabled": True})
         self._cast = (max_w, max_h)
         await self._arm_cast()
 
     async def set_viewport(self, w, h, dpr=1.0):
         """Resize Chrome's layout viewport to the panel and re-arm the screencast at the
-        matching frame size — so the page reflows to fill, and stays crisp."""
+        matching frame size — so the page reflows to fill, and stays crisp. Deduped: an
+        unchanged size is a no-op (a drag fires many identical observations)."""
         cw, ch, scale, mw, mh = viewport_metrics(w, h, dpr)
+        if (cw, ch, scale) == self._last_vp:
+            return
+        self._last_vp = (cw, ch, scale)
         await self._send("Emulation.setDeviceMetricsOverride",
                          {"width": cw, "height": ch, "deviceScaleFactor": scale, "mobile": False})
         self._cast = (mw, mh)
         await self._arm_cast()
 
     async def dispatch(self, msg: dict):
-        if msg.get("t") == "resize":
+        t = msg.get("t")
+        if t == "resize":
             await self.set_viewport(msg.get("w"), msg.get("h"), msg.get("dpr", 1))
             return
+        if t == "refresh":                 # panel became visible again → force a fresh frame
+            await self._arm_cast()
+            return
         cmd = input_to_cdp(msg)
         if cmd:
             await self._send(*cmd)

protoagent.plugin.yaml

@@ -1,6 +1,6 @@
 id: agent_browser
 name: Agent Browser
-version: 0.6.1
+version: 0.6.2
 description: >-
   Browser automation for protoAgent, backed by **agent-browser** (vercel-labs) — a
   fast native-Rust CLI/daemon that drives Chrome over CDP with accessibility-tree

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agent-browser-plugin"
-version = "0.6.1"
+version = "0.6.2"
 description = "Browser-automation plugin for protoAgent, backed by vercel-labs/agent-browser (tools + skill + workflows + an interactive, drivable browser panel)."
 requires-python = ">=3.11"
 

runtime.py

@@ -23,8 +23,18 @@ def launch_flags(cfg: dict | None) -> list[str]:
     cfg = cfg or {}
     f: list[str] = []
     headed = bool(cfg.get("headed"))
+    # Extra Chrome launch args (comma/newline separated); anti-detection + anti-throttle
+    # flags get merged in below.
+    args = [a.strip() for a in str(cfg.get("browser_args") or "").replace("\n", ",").split(",") if a.strip()]
     if headed:
         f.append("--headed")
+        # A headed window gets throttled/paused when it loses focus or is occluded, which
+        # stalls the live screencast. Keep it rendering so the panel updates even when the
+        # operator is looking at another window.
+        for a in ("--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding",
+                  "--disable-background-timer-throttling"):
+            if a not in args:
+                args.append(a)
     if str(cfg.get("profile") or "").strip():
         f += ["--profile", str(cfg["profile"]).strip()]
     if str(cfg.get("device") or "").strip():
@@ -37,10 +47,8 @@ def launch_flags(cfg: dict | None) -> list[str]:
         f += ["--max-output", str(int(cfg["max_output"]))]
 
     # ── anti-detection ──────────────────────────────────────────────────────────
-    # Extra Chrome launch args (comma/newline separated) + a UA override. `stealth` layers
-    # on the common evasions: drop the `navigator.webdriver` automation flag, and (when
-    # headless, where the UA says "HeadlessChrome") swap in a real desktop UA.
-    args = [a.strip() for a in str(cfg.get("browser_args") or "").replace("\n", ",").split(",") if a.strip()]
+    # `stealth` layers on the common evasions: drop the `navigator.webdriver` automation
+    # flag, and (when headless, where the UA says "HeadlessChrome") swap in a real desktop UA.
     ua = str(cfg.get("user_agent") or "").strip()
     if bool(cfg.get("stealth")):
         if "--disable-blink-features=AutomationControlled" not in args:

tests/test_agent_browser.py

@@ -26,10 +26,11 @@ def _toolmap(cfg=None):
 async def test_open_passes_url_and_curated_launch_flags(monkeypatch):
     rec = []
     monkeypatch.setattr(tools.subprocess, "run", fake_run(stdout="OPENED", record=rec))
-    t = _toolmap({"binary": "ab", "headed": True, "allowed_domains": "x.com", "max_output": 500})
+    # headless so argv is clean (headed injects anti-throttle --args; covered in test_runtime)
+    t = _toolmap({"binary": "ab", "allowed_domains": "x.com", "max_output": 500})
     out = await t["browser_open"].ainvoke({"url": "https://x.com"})
     assert "OPENED" in out
-    assert rec[-1] == ["ab", "--headed", "--allowed-domains", "x.com", "--max-output", "500", "open", "https://x.com"]
+    assert rec[-1] == ["ab", "--allowed-domains", "x.com", "--max-output", "500", "open", "https://x.com"]
 
 
 async def test_open_blank_url_omits_it(monkeypatch):
@@ -159,6 +160,8 @@ def test_panel_page_wires_canvas_stream_and_input():
     assert 'u.protocol==="https:" ? "wss:" : "ws:"' in html  # http→ws upgrade
     assert 'send({t:"mouse"' in html and 'send({t:"key"' in html  # input forwarding
     assert "ResizeObserver" in html and 'send({t:"resize"' in html  # responsive viewport tracking
+    assert "object-fit:contain" in html  # no distortion during resize
+    assert "visibilitychange" in html and 'send({t:"refresh"' in html  # refresh when re-shown
     assert "/api/plugins/agent_browser/nav" in html and "kit.apiFetch" in html  # nav via gated route
     assert "startBrowser" in html and 'const HOME="";' in html  # empty-state Start; blank home default
     # the removed dashboard-embed / screenshot modes leave no trace:

tests/test_runtime.py

@@ -12,12 +12,25 @@ def test_default_is_empty():
 
 
 def test_curated_flags_stable_order():
-    f = rt.launch_flags({"headed": True, "profile": "P", "device": "iPhone 16 Pro",
+    # headless keeps the argv clean (headed injects anti-throttle --args, tested separately)
+    f = rt.launch_flags({"profile": "P", "device": "iPhone 16 Pro",
                          "allowed_domains": "x.com", "confirm_actions": "nav", "max_output": 500})
-    assert f == ["--headed", "--profile", "P", "--device", "iPhone 16 Pro",
+    assert f == ["--profile", "P", "--device", "iPhone 16 Pro",
                  "--allowed-domains", "x.com", "--confirm-actions", "nav", "--max-output", "500"]
 
 
+def test_headed_injects_anti_throttle_args():
+    f = rt.launch_flags({"headed": True})
+    assert f[0] == "--headed" and f[1] == "--args"
+    aset = set(f[2].split(","))
+    assert {"--disable-backgrounding-occluded-windows", "--disable-renderer-backgrounding",
+            "--disable-background-timer-throttling"} <= aset  # keep a headed window rendering unfocused
+
+
+def test_headless_stays_clean():
+    assert rt.launch_flags({"allowed_domains": "x.com"}) == ["--allowed-domains", "x.com"]  # no --args
+
+
 def test_stealth_headless_adds_automation_arg_and_real_ua():
     f = rt.launch_flags({"stealth": True})  # headless by default
     assert f[f.index("--user-agent") + 1].startswith("Mozilla/5.0")