From 7c6220c765347f665a73621a39b66b1cd20ac426 Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Wed, 1 Jul 2026 16:02:48 -0700 Subject: [PATCH] fix(panel): stop the live dot flapping (coalesce frames + reconnect fast) + show viewport size (v0.6.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field report: "not navigating live, live dot goes offline, buggy as hell" + a narrow viewport that doesn't fill the dock. Stability (the flapping): - Coalescing frame delivery: on_frame stashes only the NEWEST frame and a sender task drains it. Under load this drops stale frames instead of letting ws.send_bytes block — the backpressure buildup was stalling the socket (frozen → proxy idle-closes → the "live" dot flaps offline → reconnect loop). - Race operator input against the CDP reader task: if Chrome's page target dies (tab gone/replaced), break and let the client reconnect to the live page (~1.2s) instead of freezing until an idle-close. Reconnect backoff 2.5s → 1.2s. Diagnostics for the size issue: - The "live" pill now shows the real viewport dims (live · WxH), so we can see what size the browser is actually rendering at (the default is 1280x577; a narrow strip means the panel-fit resize is landing small). - Resize self-corrects: re-sent ~500ms after connect in case the dock hadn't laid out yet. 47 host-free tests pass; nav re-arm + resize re-validated live; ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 11 +++++++++ browser_panel.py | 56 ++++++++++++++++++++++++++++++++---------- browser_stream.py | 7 ++++++ protoagent.plugin.yaml | 2 +- pyproject.toml | 2 +- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e277fa..772f5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v0.6.3 +- **Stop the "live" dot flapping offline.** The stream now delivers frames by **coalescing to + the newest** — under load it drops stale frames instead of letting `send` block, which was + building backpressure until the socket idle-closed and reconnected in a loop. The route also + **races operator input against the CDP socket dying**, so if the page target goes away the panel + reconnects to the live page in ~1.2s instead of freezing. Faster reconnect backoff. +- **Live viewport size in the status bar** — the "live" pill now reads `live · ×` so you can + see exactly what size the browser is rendering at (and tell whether the panel-fit is landing). +- **Resize self-corrects** — the panel re-sends its size shortly after connecting, in case the + dock hadn't finished laying out when the socket opened. + ## v0.6.2 - **Keeps live-updating when the panel isn't focused.** The screencast used to stall unless the panel had focus. Fixed on three fronts: the page is now pinned focused/visible over CDP diff --git a/browser_panel.py b/browser_panel.py index 22c7a2f..4f96705 100644 --- a/browser_panel.py +++ b/browser_panel.py @@ -94,23 +94,49 @@ async def _stream(ws: WebSocket): await ws.send_json({"t": "error", "msg": note}) await ws.close() return - dims: dict = {"wh": None} + # Coalescing delivery: on_frame just stashes the NEWEST frame and signals; a sender + # task drains it. Under load this DROPS stale frames instead of letting ws.send_bytes + # block — the backpressure buildup was what stalled the socket (frozen frames → the + # proxy idle-closes it → the "live" dot flaps offline). + latest: dict = {"jpeg": None, "wh": None} + pending = asyncio.Event() async def on_frame(jpeg: bytes, md: dict): - wh = (md.get("deviceWidth"), md.get("deviceHeight")) + latest["jpeg"] = jpeg + latest["wh"] = (md.get("deviceWidth"), md.get("deviceHeight")) + pending.set() + + async def sender(): + sent_wh = None try: - if wh != dims["wh"]: - dims["wh"] = wh - await ws.send_json({"t": "meta", "w": wh[0], "h": wh[1]}) - await ws.send_bytes(jpeg) - except Exception: # noqa: BLE001 — client vanished mid-frame; teardown follows - return + while True: + await pending.wait() + pending.clear() + jpeg, wh = latest["jpeg"], latest["wh"] + if jpeg is None: + continue + if wh != sent_wh: + sent_wh = wh + await ws.send_json({"t": "meta", "w": wh[0], "h": wh[1]}) + await ws.send_bytes(jpeg) + except Exception: # noqa: BLE001 — client gone / cancelled on teardown + pass + send_task = None try: async with browser_stream.CDPStream(page_ws, on_frame, quality=quality) as cdp: await cdp.start_screencast() + send_task = asyncio.create_task(sender()) while True: - await cdp.dispatch(await ws.receive_json()) + # race operator input against the CDP socket dying — if the page target + # goes away, break so the client reconnects to the live page fast. + recv = asyncio.create_task(ws.receive_json()) + done, _ = await asyncio.wait({recv, cdp.reader_task}, + return_when=asyncio.FIRST_COMPLETED) + if recv not in done: + recv.cancel() + break + await cdp.dispatch(recv.result()) except WebSocketDisconnect: pass except Exception: # noqa: BLE001 — a CDP/stream fault must not take down the worker @@ -119,6 +145,9 @@ async def on_frame(jpeg: bytes, md: dict): await ws.close() except Exception: # noqa: BLE001 pass + finally: + if send_task: + send_task.cancel() @router.post("/nav") async def _nav(body: dict = Body(...)): @@ -205,6 +234,7 @@ async def _nav(body: dict = Body(...)): $("url").addEventListener("keydown",(e)=>{ if(e.key==="Enter") go(); }); function setStatus(s,label){ $("dot").className="dot"+(s?(" "+s):""); $("cs").textContent=label; } +function live(){ return (devW&&devH) ? ("live · "+devW+"×"+devH) : "live"; } // show the real viewport size function showMsg(t,html){ $("mt").textContent=t; $("md").innerHTML=html||""; $("msg").style.display="flex"; } function hideMsg(){ $("msg").style.display="none"; } @@ -237,24 +267,24 @@ async def _nav(body: dict = Body(...)): const ticket=(await r.json()).ticket; ws=new WebSocket(wsUrl(ticket)); ws.binaryType="arraybuffer"; - ws.onopen=()=>{ connected=true; setStatus("ok","live"); sendResize(); }; + ws.onopen=()=>{ connected=true; setStatus("ok",live()); sendResize(); setTimeout(sendResize,500); }; ws.onmessage=onMsg; ws.onclose=()=>{ connected=false; setStatus("err","offline"); scheduleRetry(); }; ws.onerror=()=>{ try{ ws.close(); }catch(_){} }; }catch(_){ setStatus("err","offline"); scheduleRetry(); } } -function scheduleRetry(){ clearTimeout(retry); retry=setTimeout(connect,2500); } +function scheduleRetry(){ clearTimeout(retry); retry=setTimeout(connect,1200); } async function onMsg(ev){ if(typeof ev.data==="string"){ let m; try{ m=JSON.parse(ev.data); }catch(_){ return; } - if(m.t==="meta"){ if(m.w) devW=m.w; if(m.h) devH=m.h; } + if(m.t==="meta"){ if(m.w) devW=m.w; if(m.h) devH=m.h; setStatus("ok",live()); } else if(m.t==="error"){ setStatus("err","no page"); showStart(m.msg); } return; } try{ // binary → a JPEG screencast frame const bmp=await createImageBitmap(new Blob([ev.data])); if(cv.width!==bmp.width||cv.height!==bmp.height){ cv.width=bmp.width; cv.height=bmp.height; } - ctx.drawImage(bmp,0,0); bmp.close(); hideMsg(); setStatus("ok","live"); + ctx.drawImage(bmp,0,0); bmp.close(); hideMsg(); setStatus("ok",live()); }catch(_){} } diff --git a/browser_stream.py b/browser_stream.py index 627335c..3219d09 100644 --- a/browser_stream.py +++ b/browser_stream.py @@ -238,6 +238,13 @@ async def __aexit__(self, *exc): if self._ws: await self._ws.close() + @property + def reader_task(self): + """The background CDP read loop; it completes when Chrome's page socket closes + (tab gone / target replaced). The route races it so a dead stream reconnects fast + instead of freezing until an idle-close.""" + return self._reader + async def _send(self, method: str, params: dict | None = None) -> int: async with self._lock: # serialize writes: reader (acks/re-arm) + request task (input/resize) self._id += 1 diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 752e7bb..e8ba811 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,6 +1,6 @@ id: agent_browser name: Agent Browser -version: 0.6.2 +version: 0.6.3 description: >- Browser automation for protoAgent, backed by **agent-browser** (vercel-labs) — a fast native-Rust CLI/daemon that drives Chrome over CDP with accessibility-tree diff --git a/pyproject.toml b/pyproject.toml index d7c585b..2870d03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "agent-browser-plugin" -version = "0.6.2" +version = "0.6.3" description = "Browser-automation plugin for protoAgent, backed by vercel-labs/agent-browser (tools + skill + workflows + an interactive, drivable browser panel)." requires-python = ">=3.11"