From 3b10f8095deb73be9fa03a6c71c130b3651c586b Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Wed, 1 Jul 2026 00:41:57 -0700 Subject: [PATCH 1/2] =?UTF-8?q?feat(panel):=20fully=20interactive=20CDP-sc?= =?UTF-8?q?reencast=20viewport=20=E2=80=94=20full=20switchover=20(v0.6.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the entire dashboard-embed / screenshot panel approach with a single fully drivable browser viewport. A live CDP screencast (event-driven JPEG frames) is painted on a and operator mouse/keyboard/scroll are forwarded back via Input.dispatch*, all over a gated same-origin WebSocket — so it works on the host AND a remote fleet member, and you can drive the page alongside the agent. New: - browser_stream.py — a second-CDP-client bridge (Page.startScreencast out, input in) + single-use ticket auth. resolve_page_target/pick_page_target/input_to_cdp are pure + host-free-tested. - browser_panel.py — the interactive canvas page + gated POST /stream-ticket and the self-gated WS /stream. WS auth: the host's operator-bearer middleware is HTTP-only and skips WS handshakes, so the panel mints a ticket over the gated HTTP route and presents it on the WS URL; the handler validates + burns it. Full switchover (no backward compatibility): - Removed panel_mode (interactive is the only panel), the screenshot `minimal` mode and /shot, and the `full` dashboard-embed page. - Removed the browser_dashboard tool (16 tools now), lifecycle.py (dashboard daemon boot/shutdown surface), and the /dashboard control routes. - Removed config: panel_mode, dashboard_port, manage_dashboard. Validated end-to-end against the real agent-browser binary through the production routes (bad ticket refused, ticket minted, real frame streamed, keys + click land). 34 host-free tests pass; ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 22 ++ README.md | 35 ++- __init__.py | 33 +-- browser_panel.py | 414 ++++++++++++++--------------------- browser_stream.py | 236 ++++++++++++++++++++ lifecycle.py | 54 ----- protoagent.plugin.yaml | 24 +- pyproject.toml | 4 +- requirements-dev.txt | 6 +- requirements.txt | 5 +- tests/test_agent_browser.py | 152 +++++-------- tests/test_browser_stream.py | 166 ++++++++++++++ tools.py | 16 +- 13 files changed, 686 insertions(+), 481 deletions(-) create mode 100644 browser_stream.py delete mode 100644 lifecycle.py create mode 100644 tests/test_browser_stream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2855864..316ef2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## v0.6.0 +- **The Browser panel is now a fully interactive, drivable viewport — and it is the ONLY mode.** + A live **CDP screencast** — event-driven JPEG frames (not a screenshot poll) painted on a + `` — forwards your **mouse, keyboard, and scroll** back into the page via + `Input.dispatch*`. You can click, type, and scroll the real browser from the console, right + alongside the agent. Because every byte is a **gated same-origin WebSocket**, it works on the + host **and** a remote fleet member. +- **How it works.** A second CDP client attaches to the same Chrome agent-browser drives (via + `agent-browser get cdp-url`); `browser_stream.py` bridges `Page.startScreencast` ⇆ the panel. + The nav toolbar (url / back / forward / reload) reuses the gated HTTP `/nav` route. +- **WebSocket auth.** The host's operator-bearer gate is HTTP-only and does **not** cover WS + handshakes, so the stream self-gates: the panel mints a **single-use ticket** from the gated + `POST /stream-ticket` (bearer-checked) and presents it on the WS URL; the handler validates and + burns it. Safe in gated deployments, transparent in open ones. +- **Full switchover — the old dashboard-embed approach is removed** (no backward compatibility): + - `panel_mode` is gone (there is one panel now); so is the screenshot `minimal` mode and its + `/shot` route, and the `full` dashboard-embed page. + - The **`browser_dashboard` tool is removed** (16 tools now) along with the boot/shutdown + **dashboard lifecycle** (`lifecycle.py`) and the panel's `/dashboard` control routes. + - Removed config: `panel_mode`, `dashboard_port`, `manage_dashboard`. The interactive panel + talks CDP directly and never needs agent-browser's separate dashboard daemon. + ## v0.5.1 - **Full mode: an "Open ↗" button** to pop the dashboard out into a full browser tab, alongside the inline embed. Shown whenever the dashboard is on this machine (loopback host) — including over an diff --git a/README.md b/README.md index 0f1880b..dba7d2e 100644 --- a/README.md +++ b/README.md @@ -33,25 +33,21 @@ point. `browser_snapshot`, `browser_click`, `browser_fill`, `browser_type`, `browser_get_text`/`get_html`/`get_value`, `browser_press`, `browser_hover`, `browser_eval`, `browser_screenshot`, `browser_back`/`forward`/`reload`, - `browser_close`, `browser_dashboard`. + `browser_close`. - **Skill** — a discovery skill that defers to the CLI's always-current workflow content (`agent-browser skills get core`), so instructions never go stale. - **Workflows** — declarative browser recipes (browse-and-extract, fill-a-form, …). -- **Browser panel** — a console view (ADR 0026) for watching/driving the browser. Two - modes, set by `panel_mode`: - - **`full` (default)** — **embeds agent-browser's dashboard inline** (viewport + activity/ - console/network feeds) at its own **local origin** (`http://:/`). Best for a - **local setup** (console + agent-browser on one machine). Because the dashboard is a - Next.js app with root-absolute assets (no base-path), it only loads at its own origin — - so when the console is opened **remotely** (a fleet member, a non-loopback host, or over - https) its `localhost` isn't reachable from your browser, and the panel shows a **clear - error** (pointing you at `minimal`) rather than a blank frame. - - **`minimal`** — a **live screenshot** of the viewport + a nav toolbar, all through the - **gated same-origin routes**. Works everywhere (host and member), no dashboard daemon - needed — use it for a remote/member agent. - - Either mode can **start the dashboard from the panel** (no terminal) — the Start/Stop - control hits the gated `POST /api/plugins/agent_browser/dashboard`. +- **Browser panel** — a console view (ADR 0026) that is a **fully drivable viewport**. A live + **CDP screencast** (event-driven JPEG frames, not a screenshot poll) is painted on a + ``, and your **mouse / keyboard / scroll** are forwarded back into the page via + `Input.dispatch*` — so you can click, type, and scroll the real browser from the console, + alongside the agent. Everything rides a **gated same-origin WebSocket**, so it works on the + **host and a remote fleet member** alike. A second CDP client attaches to the same Chrome + agent-browser drives (`agent-browser get cdp-url`); `browser_stream.py` does the bridging. + + **WebSocket auth:** the host's operator-bearer gate is HTTP-only and doesn't cover WS + handshakes, so the stream self-gates — the panel mints a **single-use ticket** from the gated + `POST /api/plugins/agent_browser/stream-ticket` and presents it on the WS URL. ## Requirements @@ -75,7 +71,6 @@ plugins: agent_browser: binary: agent-browser - dashboard_port: 4848 ``` ## Layout @@ -83,12 +78,12 @@ agent_browser: | File | What | |---|---| | `tools.py` | the browser tools — subprocess wrappers over the `agent-browser` CLI | -| `browser_panel.py` | the Browser panel — `minimal` (live viewport + nav + dashboard control) and `full` (a launcher for the dashboard) | -| `lifecycle.py` | the dashboard daemon surface — start on boot, stop on shutdown (ADR 0018) | +| `browser_panel.py` | the Browser panel page + routes — the interactive canvas + the gated nav / stream-ticket / WS-stream routes | +| `browser_stream.py` | the CDP bridge — screencast frames out, input in; the WS ticket auth | | `skills/` | the discovery skill (defers to `agent-browser skills get core`) | | `workflows/` | declarative browser recipes | | `tests/` | the host-free pytest suite (subprocess mocked — no binary needed) | -| `__init__.py` | `register()` — wires tools + panel + lifecycle; skills/workflows auto-discovered | +| `__init__.py` | `register()` — wires tools + the interactive panel; skills/workflows auto-discovered | The operator knobs (panel mode, headed, allowed domains, profile, device, …) are editable in **Settings ▸ Plugins ▸ Agent Browser**, or under `agent_browser:` in `langgraph-config.yaml`. diff --git a/__init__.py b/__init__.py index 402e351..fb26ce3 100644 --- a/__init__.py +++ b/__init__.py @@ -2,9 +2,10 @@ Composition over construction: this plugin is a thin shell over the `agent-browser` CLI/daemon. It contributes the browser **tools** (subprocess wrappers), a discovery -**skill** + browser **workflows** (auto-discovered from skills/ and workflows/), and a -**Browser panel** console view that embeds agent-browser's own live dashboard — it -does NOT reimplement browser automation or a renderer. +**skill** + browser **workflows** (auto-discovered from skills/ and workflows/), and an +interactive **Browser panel** console view — a live, drivable CDP-screencast viewport +(browser_stream bridges Chrome's CDP to a canvas over a gated WebSocket). It does NOT +reimplement browser automation or a renderer. Ships DISABLED. Enable with `plugins: { enabled: [agent_browser] }` and put the `agent-browser` binary on PATH (`npm i -g agent-browser && agent-browser install`). @@ -28,12 +29,13 @@ def register(registry) -> None: except Exception: # noqa: BLE001 — tools are the foundation; log loudly if they fail log.exception("[agent_browser] registering browser tools failed") - # Browser panel console view (embeds agent-browser's dashboard). Built out by the - # board; register it best-effort so the foundation works before the view lands. - # TWO routers at DISTINCT prefixes: the PAGE (+ the iframe-loaded /panel/dash - # proxy, which can't carry a bearer) stays on the public /plugins/agent_browser; - # the shot/nav DATA routes mount under /api/plugins/agent_browser so they - # inherit the operator bearer gate (plugin-view rule 2). + # Interactive Browser panel console view. Register it best-effort so the tools still + # serve if the panel can't import. TWO routers at DISTINCT prefixes: the PAGE stays on + # the public /plugins/agent_browser (an iframe page-load can't carry a bearer); the + # DATA routes (the nav toolbar, the stream ticket + the /stream WS) mount under + # /api/plugins/agent_browser so the HTTP ones inherit the operator bearer gate + # (plugin-view rule 2). The WS gates itself with a single-use ticket — the host's auth + # middleware doesn't cover WS handshakes. try: from .browser_panel import build_panel_data_router, build_panel_router registry.register_router(build_panel_router(cfg)) @@ -43,16 +45,5 @@ def register(registry) -> None: except Exception: # noqa: BLE001 log.exception("[agent_browser] mounting browser panel failed") - # Lifecycle (ADR 0018): on shutdown, stop the dashboard daemon we manage so it - # doesn't outlive the server (dashboard-only; the session is left alone). - try: - from .lifecycle import make_dashboard_surface - start, stop = make_dashboard_surface(cfg) - registry.register_surface(start, stop=stop, name="agent-browser-dashboard") - except Exception: # noqa: BLE001 — lifecycle is best-effort; tools/panel still serve - log.exception("[agent_browser] registering lifecycle surface failed") - # skills/ and workflows/ are auto-discovered (ADR 0027) — no register call. - log.info("[agent_browser] registered browser tools (binary=%s, dashboard:%s, manage=%s)", - cfg.get("binary", "agent-browser"), cfg.get("dashboard_port", 4848), - cfg.get("manage_dashboard", True)) + log.info("[agent_browser] registered browser tools (binary=%s)", cfg.get("binary", "agent-browser")) diff --git a/browser_panel.py b/browser_panel.py index 512bebd..f03e02b 100644 --- a/browser_panel.py +++ b/browser_panel.py @@ -1,70 +1,55 @@ -"""Browser panel console view (ADR 0026) — two modes, set by ``panel_mode``. +"""Browser panel console view (ADR 0026) — a **fully drivable** browser viewport. -- ``minimal`` (default): a viewport-only page we render ourselves — a live screenshot - (polled from the CLI through the gated /shot route, same-origin, no WS/proxy - dependency) + a nav toolbar (url / back / forward / reload) + a Dashboard control - (start/stop/status). Works everywhere — host and member alike. -- ``full``: a launcher for agent-browser's own dashboard (viewport + activity/console/ - network feeds). We deliberately do NOT embed it: that dashboard is a prebuilt Next.js - app whose assets are ROOT-ABSOLUTE (``/_next/…``) with no base-path option, so it can't - render under a sub-path reverse proxy (the long-standing blank panel) — it only renders - at its OWN origin. Full mode opens it there ("Open dashboard ↗"), which works on a - local/host setup; for a remote member, use minimal. +A live **CDP screencast** (event-driven JPEG frames, not a screenshot poll) is bridged +from the agent-browser Chrome to a ```` over a **gated same-origin WebSocket**, +and operator mouse / keyboard / scroll are forwarded back via ``Input.dispatch*`` — so +you can actually click, type, and scroll the page, alongside the agent. It rides the +fleet proxy (all bytes are same-origin), so it works on the host AND a remote member. +Streaming/input lives in ``browser_stream``; this file is the page + the routes. Self-contained vanilla JS (no build step). Every fetch / link is derived from ``base = location.pathname.split("/plugins/")[0]`` (="" on the host, ``/agents/`` -when proxied) so the page is same-origin + slug-aware (the token/theme handshake). The -browser is driven only through the CLI behind same-origin routes; the agent's own -browser_* tools are the primary driver — this is an operator viewport. +when proxied) so the page is same-origin + slug-aware (the token/theme handshake). """ from __future__ import annotations import asyncio import logging -import os import subprocess -import tempfile -import time # Module-scope fastapi imports (the host always provides fastapi; __init__.py catches # ImportError so the tools still serve if the panel can't import). -from fastapi import APIRouter, Body -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Response +from fastapi import APIRouter, Body, WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse, JSONResponse -log = logging.getLogger("protoagent.plugins.agent_browser") +from . import browser_stream -_SHOT_PATH = os.path.join(tempfile.gettempdir(), "agent_browser_panel.png") -_shot_ts = 0.0 # last successful capture (cheap throttle so polling can't storm the CLI) +log = logging.getLogger("protoagent.plugins.agent_browser") def build_panel_router(cfg: dict | None): - - cfg = cfg or {} - port = int(cfg.get("dashboard_port", 4848)) - mode = str(cfg.get("panel_mode", "minimal")).strip().lower() - router = APIRouter() @router.get("/panel") async def _panel(): - page = _MINIMAL_PAGE if mode == "minimal" else _FULL_PAGE - return HTMLResponse(page.replace("__DASH_PORT__", str(port))) + return HTMLResponse(_INTERACTIVE_PAGE) return router def build_panel_data_router(cfg: dict | None): - """The minimal-mode DATA/ACTION routes — mounted under - ``/api/plugins/agent_browser`` so they inherit the operator bearer gate - (plugin-view rule 2). Previously ``/panel/shot`` + ``POST /panel/nav`` lived - under the public ``/plugins/`` prefix: on a token-gated deployment anyone who - could reach the port could DRIVE the operator's browser session and read its - screen without the bearer.""" + """The panel DATA/ACTION routes — mounted under ``/api/plugins/agent_browser``. + The HTTP routes (``POST /nav``, ``POST /stream-ticket``) inherit the operator bearer + gate (plugin-view rule 2), so nobody who lacks the bearer can drive the browser or + mint a stream ticket. + + ``WS /stream`` is the exception the host forces on us: its auth middleware is + HTTP-only and does NOT cover WebSocket handshakes, so the WS gates itself with the + single-use ticket that ``POST /stream-ticket`` (which IS gated) hands out.""" cfg = cfg or {} binary = str(cfg.get("binary") or "agent-browser") timeout = float(cfg.get("timeout_s", 60)) - port = int(cfg.get("dashboard_port", 4848)) router = APIRouter() @@ -77,20 +62,53 @@ def _run(*args: str) -> tuple[int, str]: except subprocess.TimeoutExpired: return 124, "timed out" - @router.get("/shot") - async def _shot(): - """Latest viewport as a PNG. Throttled to ~1/0.8s so a fast poller can't - spawn a screenshot subprocess per frame.""" - global _shot_ts - now = time.monotonic() - if now - _shot_ts > 0.8 or not os.path.exists(_SHOT_PATH): - rc, err = await asyncio.to_thread(lambda: _run("screenshot", _SHOT_PATH)) - if rc == 0 and os.path.exists(_SHOT_PATH): - _shot_ts = now - else: - return Response(status_code=503, content=f"no frame: {err[:200]}") - return FileResponse(_SHOT_PATH, media_type="image/png", - headers={"Cache-Control": "no-store"}) + # ── interactive stream: a single-use ticket (gated) + the WS bridge (self-gated) ── + @router.post("/stream-ticket") + async def _stream_ticket(): + """Mint a single-use ticket for the interactive WS. HTTP, so it rides the + host's operator-bearer gate — only an authenticated console reaches it.""" + return JSONResponse({"ticket": browser_stream.mint_ticket()}) + + @router.websocket("/stream") + async def _stream(ws: WebSocket): + """Interactive viewport: CDP screencast frames out (binary JPEG), operator + input in (JSON → ``Input.dispatch*``). Ticket-gated (the host doesn't gate + WS). One sender only — ``on_frame`` — while the loop just receives, so the two + directions never race on the socket.""" + if not browser_stream.consume_ticket(ws.query_params.get("ticket", "")): + await ws.close(code=1008) # policy violation: missing/replayed/expired ticket + return + await ws.accept() + page_ws, note = await asyncio.to_thread(browser_stream.resolve_page_target, binary, timeout) + if not page_ws: + await ws.send_json({"t": "error", "msg": note}) + await ws.close() + return + dims: dict = {"wh": None} + + async def on_frame(jpeg: bytes, md: dict): + wh = (md.get("deviceWidth"), md.get("deviceHeight")) + try: + if wh != dims["wh"]: + dims["wh"] = wh + await ws.send_json({"t": "meta", "w": wh[0], "h": wh[1]}) + await ws.send_bytes(jpeg) + except Exception: # noqa: BLE001 — client vanished mid-frame; teardown follows + return + + try: + async with browser_stream.CDPStream(page_ws, on_frame) as cdp: + await cdp.start_screencast() + while True: + await cdp.dispatch(await ws.receive_json()) + except WebSocketDisconnect: + pass + except Exception: # noqa: BLE001 — a CDP/stream fault must not take down the worker + log.exception("[agent_browser] interactive stream failed") + try: + await ws.close() + except Exception: # noqa: BLE001 + pass @router.post("/nav") async def _nav(body: dict = Body(...)): @@ -107,169 +125,42 @@ async def _nav(body: dict = Body(...)): return JSONResponse({"ok": False, "error": f"bad action {action!r}"}) return JSONResponse({"ok": rc == 0, "error": "" if rc == 0 else err[:200]}) - # ── dashboard control (start it up entirely from the panel UI) ────────────── - # The agent-browser dashboard is a standalone daemon. These let the panel show its - # status and start/stop it without dropping to a terminal. Status is a live probe of - # the daemon's loopback port (there's no `dashboard status` subcommand). - @router.get("/dashboard") - async def _dash_status(): - try: - import httpx - - async with httpx.AsyncClient(timeout=httpx.Timeout(2.0)) as c: - await c.get(f"http://127.0.0.1:{port}/") - running = True - except Exception: # noqa: BLE001 — connection refused / down → not running - running = False - return JSONResponse({"running": running, "port": port}) - - @router.post("/dashboard") - async def _dash_control(body: dict = Body(...)): - """Start or stop the dashboard daemon from the UI. `action` is start|stop.""" - action = str(body.get("action", "")).strip().lower() - if action == "start": - rc, err = await asyncio.to_thread(lambda: _run("dashboard", "start", "--port", str(port))) - elif action == "stop": - rc, err = await asyncio.to_thread(lambda: _run("dashboard", "stop")) - else: - return JSONResponse({"ok": False, "error": f"action must be start|stop, got {action!r}"}) - return JSONResponse({"ok": rc == 0, "error": "" if rc == 0 else err[:200], "port": port}) - return router -# ── full mode (default): embed agent-browser's dashboard at its OWN origin ────── -# The dashboard is a Next.js app with root-absolute assets (no base-path), so it only -# renders at its own origin (http://:/) — never under a sub-path proxy. Full -# mode therefore ASSUMES A LOCAL setup (console + agent-browser on one machine) and iframes -# that loopback origin directly. When the console is opened remotely (a fleet member, or a -# non-loopback host, or over https) that loopback dashboard isn't reachable from the -# operator's browser — we DETECT that and show a clear error instead of a blank frame. -_FULL_PAGE = r""" +# ── the interactive browser panel: a live, drivable CDP-screencast viewport ───── +# A fed JPEG frames over the gated WS (browser_stream bridges CDP screencast), +# with mouse/keyboard/scroll forwarded back as Input.dispatch*. Chrome is the protoLabs +# design system (plugin-kit CSS + .pl-* components); the viewport canvas itself is a +# bespoke domain surface (theme the frame around it, not the pixels). +_INTERACTIVE_PAGE = r""" Browser -
Browser - -
-
- -
-
-""" - - -# ── minimal mode: viewport-only (screenshot-poll + nav toolbar) ──────────────── -# Chrome is the protoLabs design system: plugin-kit CSS + .pl-* components (nav as -# .pl-btn, url as .pl-input, empty state as .pl-empty), themed live by the handshake. -_MINIMAL_PAGE = r""" -Browser - - -
@@ -277,78 +168,99 @@ async def _dash_control(body: dict = Body(...)): - + connecting…
-
-
-
No page loaded
-
Open a URL above, or let the agent drive — browser_open.
-
+
+ +
""" diff --git a/browser_stream.py b/browser_stream.py new file mode 100644 index 0000000..b990dd8 --- /dev/null +++ b/browser_stream.py @@ -0,0 +1,236 @@ +"""Interactive browser streaming — a CDP screencast + input bridge. + +The `interactive` panel mode renders a live, *drivable* viewport instead of a +screenshot poll. It works by attaching a **second CDP client** to the same Chrome +that agent-browser drives (the CLI hands us the endpoint via `agent-browser get +cdp-url`), running **`Page.startScreencast`** for event-driven JPEG frames, and +forwarding operator input back with **`Input.dispatch*`**. A second CDP client +coexists with agent-browser's own session — the agent and the operator can both +touch the page. + +Everything is bridged to the panel over a **gated same-origin WebSocket** +(`/api/plugins/agent_browser/stream`), so it inherits the operator bearer gate and +rides the fleet proxy — the interactive viewport works on a remote member, not just +the host (the thing the `full` dashboard embed never could). + +This module is split so the CDP-facing brains are pure and host-free-testable: +``_http_base_from_ws`` / ``pick_page_target`` / ``input_to_cdp`` have no IO. The +``CDPStream`` async client and ``resolve_page_target`` do the talking; the WebSocket +route lives in ``browser_panel``. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import secrets +import subprocess +import time +import urllib.request + +log = logging.getLogger("protoagent.plugins.agent_browser") + + +# ── WebSocket auth: single-use tickets ───────────────────────────────────────── +# The host's operator-bearer gate is HTTP-only middleware — it does NOT cover +# WebSocket handshakes (a browser `new WebSocket()` can't send an Authorization +# header anyway). So the stream WS gates itself: the panel first calls the *gated* +# `POST /stream-ticket` (bearer-checked by the host, so only an authenticated +# console gets one), then presents the ticket on the WS URL. This mirrors the host's +# own SSE-token escape hatch for `/api/events`. Short-lived + single-use so a ticket +# leaked into a proxy/access log is near-worthless. +_TICKET_TTL = 30.0 +_tickets: dict[str, float] = {} + + +def _prune_tickets(now: float) -> None: + for k in [k for k, exp in _tickets.items() if exp < now]: + _tickets.pop(k, None) + + +def mint_ticket() -> str: + """Issue a single-use ticket good for ~30s. Called only from the gated HTTP + route, so possession of a ticket proves the caller cleared the operator gate.""" + now = time.monotonic() + _prune_tickets(now) + t = secrets.token_urlsafe(24) + _tickets[t] = now + _TICKET_TTL + return t + + +def consume_ticket(ticket: str) -> bool: + """Validate + burn a ticket. False if unknown/expired (→ reject the WS).""" + now = time.monotonic() + _prune_tickets(now) + exp = _tickets.pop(ticket, None) if ticket else None + return exp is not None and exp >= now + + +# ── pure helpers (host-free-testable — no IO) ────────────────────────────────── + +def _http_base_from_ws(ws_url: str) -> str: + """`ws://127.0.0.1:52886/devtools/browser/…` → `http://127.0.0.1:52886`. The + CDP HTTP endpoints (`/json/list`, `/json/version`) live at the same host:port.""" + rest = ws_url.split("://", 1)[-1] + authority = rest.split("/", 1)[0] + return "http://" + authority + + +def pick_page_target(targets: list[dict], current_url: str = "") -> str | None: + """Choose which CDP target to stream from a `/json/list` array. Prefer the + `page` matching the session's current URL (the active tab); else the first real + `page` (skipping chrome:// / devtools surfaces). Returns its + `webSocketDebuggerUrl`, or None if there's no page to stream.""" + pages = [t for t in targets + if t.get("type") == "page" and t.get("webSocketDebuggerUrl")] + if not pages: + return None + cur = (current_url or "").strip() + if cur: + for t in pages: + if t.get("url", "") == cur: + return t["webSocketDebuggerUrl"] + real = [t for t in pages if not t.get("url", "").startswith(("chrome://", "devtools://"))] + return (real or pages)[0]["webSocketDebuggerUrl"] + + +# Modifier bit-mask CDP expects on Input events (Alt=1, Ctrl=2, Meta=4, Shift=8). +def _modifiers(m: dict) -> int: + return ((1 if m.get("alt") else 0) | (2 if m.get("ctrl") else 0) + | (4 if m.get("meta") else 0) | (8 if m.get("shift") else 0)) + + +_MOUSE_TYPE = {"down": "mousePressed", "up": "mouseReleased", "move": "mouseMoved"} + + +def input_to_cdp(msg: dict) -> tuple[str, dict] | None: + """Translate a panel input message → a `(cdp_method, params)` pair, or None if + it isn't a drivable input. Coordinates arrive already in CSS pixels (the client + maps canvas→page against the frame metadata), so we pass them straight through. + + Panel messages: + {t:"mouse", action:"down|up|move", x, y, button?, clickCount?, buttons?, mods…} + {t:"wheel", x, y, dx, dy, mods…} + {t:"key", action:"down|up", key, code?, text?, keyCode?, mods…} + """ + t = msg.get("t") + if t == "mouse": + typ = _MOUSE_TYPE.get(msg.get("action", "")) + if not typ: + return None + p = {"type": typ, "x": float(msg.get("x", 0)), "y": float(msg.get("y", 0)), + "modifiers": _modifiers(msg)} + if typ != "mouseMoved": + p["button"] = msg.get("button", "left") + p["clickCount"] = int(msg.get("clickCount", 1)) + p["buttons"] = int(msg.get("buttons", 0)) + return "Input.dispatchMouseEvent", p + if t == "wheel": + return "Input.dispatchMouseEvent", { + "type": "mouseWheel", "x": float(msg.get("x", 0)), "y": float(msg.get("y", 0)), + "deltaX": float(msg.get("dx", 0)), "deltaY": float(msg.get("dy", 0)), + "modifiers": _modifiers(msg)} + if t == "key": + typ = {"down": "keyDown", "up": "keyUp"}.get(msg.get("action", "")) + if not typ: + return None + text = msg.get("text", "") + # A keyDown that produces a character must be dispatched as "keyDown" with + # text; CDP turns text-bearing keyDowns into the actual input. + p = {"type": typ, "key": msg.get("key", ""), "code": msg.get("code", ""), + "modifiers": _modifiers(msg)} + if msg.get("keyCode"): + p["windowsVirtualKeyCode"] = int(msg["keyCode"]) + p["nativeVirtualKeyCode"] = int(msg["keyCode"]) + if typ == "keyDown" and text: + p["text"] = text + return "Input.dispatchKeyEvent", p + return None + + +# ── the CDP client (IO) ──────────────────────────────────────────────────────── + +def resolve_page_target(binary: str, timeout: float = 10.0) -> tuple[str | None, str]: + """Ask agent-browser for its Chrome CDP endpoint, then find the active page's + per-target WebSocket. Returns ``(page_ws_url | None, note)`` — note carries a + human-readable reason when there's nothing to stream (no session / no page).""" + try: + cdp = subprocess.run([binary, "get", "cdp-url"], capture_output=True, + text=True, timeout=timeout) + except FileNotFoundError: + return None, f"{binary!r} not on PATH" + except subprocess.TimeoutExpired: + return None, "agent-browser get cdp-url timed out" + browser_ws = (cdp.stdout or "").strip().splitlines()[0].strip() if cdp.stdout else "" + if cdp.returncode != 0 or not browser_ws.startswith("ws"): + return None, ((cdp.stderr or "").strip() or "no CDP url — is a session open?") + base = _http_base_from_ws(browser_ws) + cur = "" + try: + u = subprocess.run([binary, "get", "url"], capture_output=True, text=True, timeout=timeout) + cur = (u.stdout or "").strip().splitlines()[0].strip() if u.returncode == 0 else "" + except Exception: # noqa: BLE001 — current url is a nicety for tab selection + cur = "" + try: + with urllib.request.urlopen(base + "/json/list", timeout=timeout) as r: + targets = json.loads(r.read().decode()) + except Exception as e: # noqa: BLE001 + return None, f"CDP /json/list unreachable at {base}: {e}" + page = pick_page_target(targets, cur) + return (page, "" if page else "no page target to stream (open a URL first)") + + +class CDPStream: + """A minimal async CDP client over one page target: start a screencast, ack + frames, and dispatch input. `frame_cb(jpeg_bytes, metadata)` is called for each + `Page.screencastFrame`. Requires the ``websockets`` package (a uvicorn extra — + already present wherever the host serves WebSockets).""" + + def __init__(self, page_ws_url: str, frame_cb): + self._url = page_ws_url + self._frame_cb = frame_cb + self._ws = None + self._id = 0 + self._reader: asyncio.Task | None = None + + async def __aenter__(self): + import websockets + self._ws = await websockets.connect(self._url, max_size=None, open_timeout=10) + self._reader = asyncio.create_task(self._read_loop()) + return self + + async def __aexit__(self, *exc): + if self._reader: + self._reader.cancel() + if self._ws: + await self._ws.close() + + async def _send(self, method: str, params: dict | None = None) -> int: + self._id += 1 + await self._ws.send(json.dumps({"id": self._id, "method": method, "params": params or {}})) + return self._id + + async def start_screencast(self, max_w: int = 1280, max_h: int = 800, quality: int = 60): + await self._send("Page.enable") + await self._send("Page.startScreencast", {"format": "jpeg", "quality": quality, + "maxWidth": max_w, "maxHeight": max_h, "everyNthFrame": 1}) + + async def dispatch(self, msg: dict): + cmd = input_to_cdp(msg) + if cmd: + await self._send(*cmd) + + async def _read_loop(self): + import base64 + async for raw in self._ws: + try: + m = json.loads(raw) + except Exception: # noqa: BLE001 + continue + if m.get("method") == "Page.screencastFrame": + p = m["params"] + try: + await self._frame_cb(base64.b64decode(p["data"]), p.get("metadata", {})) + finally: + await self._send("Page.screencastFrameAck", {"sessionId": p["sessionId"]}) diff --git a/lifecycle.py b/lifecycle.py deleted file mode 100644 index bb96fc2..0000000 --- a/lifecycle.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Plugin lifecycle (ADR 0018) — tear down the dashboard daemon on shutdown. - -agent-browser's dashboard runs as a standalone background daemon. Without a stop -hook it outlives the agent (orphaned on every restart). This registers a surface -whose ``stop`` best-effort runs ``agent-browser dashboard stop`` so the daemon dies -with the server. Dashboard-only by design — the browser session is left alone. - -Gated by ``manage_dashboard`` (default true): set it false for a shared/persistent -dashboard you don't want the agent to stop. -""" - -from __future__ import annotations - -import asyncio -import logging -import subprocess - -log = logging.getLogger("protoagent.plugins.agent_browser") - - -def make_dashboard_surface(cfg: dict | None): - """Return ``(start, stop)`` for ``register_surface`` — full ownership of the - managed dashboard: ``start`` launches it on boot, ``stop`` tears it down on - shutdown. Both best-effort and gated by ``manage_dashboard`` (so a shared/ - persistent dashboard, ``manage_dashboard: false``, is left untouched).""" - cfg = cfg or {} - binary = str(cfg.get("binary") or "agent-browser") - manage = bool(cfg.get("manage_dashboard", True)) - port = int(cfg.get("dashboard_port", 4848)) - - async def start(): - if not manage: - return None - try: - await asyncio.to_thread( - lambda: subprocess.run([binary, "dashboard", "start", "--port", str(port)], - capture_output=True, text=True, timeout=30)) - log.info("[agent_browser] dashboard daemon started on :%s", port) - except Exception: # noqa: BLE001 — best-effort; the binary may not be installed yet - log.warning("[agent_browser] dashboard start on boot failed", exc_info=True) - return None - - async def stop(): - if not manage: - return - try: - await asyncio.to_thread( - lambda: subprocess.run([binary, "dashboard", "stop"], - capture_output=True, text=True, timeout=15)) - log.info("[agent_browser] dashboard daemon stopped on shutdown") - except Exception: # noqa: BLE001 — teardown is best-effort; never block shutdown - log.warning("[agent_browser] dashboard stop on shutdown failed", exc_info=True) - - return start, stop diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 04bfdfa..84196e7 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -1,14 +1,15 @@ id: agent_browser name: Agent Browser -version: 0.5.1 +version: 0.6.0 description: >- Browser automation for protoAgent, backed by **agent-browser** (vercel-labs) — a fast native-Rust CLI/daemon that drives Chrome over CDP with accessibility-tree snapshots and compact `@eN` element refs. Wraps the CLI as tools (open, snapshot, click, fill, type, screenshot, …), ships a discovery skill that defers to the CLI's always-current workflow content, browser workflows, and a **Browser panel** - console view that embeds agent-browser's own live dashboard (the viewport + - activity/console/network feeds). + console view — a fully interactive, drivable viewport (live CDP screencast + mouse/ + keyboard/scroll over a gated same-origin WebSocket), so you can click and type in the + page alongside the agent. Requires the **`agent-browser`** binary on PATH (`npm i -g agent-browser && agent-browser install`). Ships DISABLED; enable with @@ -21,17 +22,7 @@ min_protoagent_version: "0.27.0" config_section: agent_browser config: binary: agent-browser # the agent-browser CLI on PATH (override for a pinned path) - dashboard_port: 4848 # agent-browser dashboard start --port; full-mode panel iframes this timeout_s: 60 # per-command subprocess timeout - panel_mode: full # Browser panel layout. `full` (default): embed agent-browser's own - # dashboard (viewport + activity/console/network feeds) at its local - # origin — best for a LOCAL setup (console + agent-browser on one box). - # When the console is opened remotely (a fleet member, a non-loopback - # host, or over https) the dashboard's localhost isn't reachable from - # your browser, so the panel shows a clear error — use `minimal` there. - # `minimal`: a screenshot viewport + nav toolbar driven through the - # gated same-origin routes; works everywhere, no daemon needed. Both - # modes can start/stop the dashboard from the panel. # Runtime / launch options — a curated set passed to `agent-browser open` so you can # shape and lock down the browser the agent spins up. Blank/0/false = CLI default. headed: false # show a real browser window instead of headless @@ -40,24 +31,19 @@ config: allowed_domains: "" # comma-separated navigable-domain whitelist (e.g. "example.com,*.foo.com") confirm_actions: "" # comma-separated action categories that require confirmation max_output: 0 # cap page-text output chars (LLM-safety); 0 = CLI default - manage_dashboard: true # own the dashboard lifecycle (ADR 0018): start it on boot, stop it on - # shutdown. Set false to leave a shared/persistent dashboard untouched. # Editable in Settings ▸ Plugins (ADR 0019) — the operator knobs above as UI fields. settings: - - { key: panel_mode, label: "Browser panel mode", type: select, options: [full, minimal], description: "full (default): embed agent-browser's dashboard (viewport + activity/console/network feeds) at its local origin — for a local setup (console + agent-browser on one machine). Opened remotely (a fleet member / non-loopback host / https), it shows a clear error instead of a blank frame. minimal: a screenshot viewport + nav toolbar via gated same-origin routes — works everywhere, no daemon. Both can start/stop the dashboard from the panel." } - { key: headed, label: "Headed browser", type: bool, description: "Show a real browser window instead of running headless." } - { key: allowed_domains, label: "Allowed domains", type: string, description: "Comma-separated navigable-domain allowlist (e.g. example.com,*.foo.com). Blank = unrestricted." } - { key: confirm_actions, label: "Confirm actions", type: string, description: "Comma-separated action categories that require confirmation before the agent runs them." } - { key: profile, label: "Browser profile dir", type: string, description: "Path to a browser profile directory — isolation + persisted auth/cookies across sessions." } - { key: device, label: "Device emulation", type: string, description: "Emulate a device, e.g. \"iPhone 16 Pro\". Blank = desktop." } - { key: max_output, label: "Max page-text output (chars)", type: number, description: "Cap characters of page text returned to the model (LLM-safety). 0 = the CLI default." } - - { key: dashboard_port, label: "Dashboard port", type: number, description: "Port for agent-browser's dashboard daemon; the full-mode panel reverse-proxies it." } - { key: timeout_s, label: "Command timeout (s)", type: number, description: "Per-command subprocess timeout for the browser tools." } - - { key: manage_dashboard, label: "Manage dashboard lifecycle", type: bool, description: "Start the dashboard on boot and stop it on shutdown. Turn off to leave a shared/persistent dashboard untouched." } - { key: binary, label: "agent-browser binary", type: string, description: "The agent-browser CLI on PATH (override with a pinned absolute path if needed)." } -# Console view (ADR 0026) — embeds agent-browser's live dashboard (viewport + feeds). +# Console view (ADR 0026) — an interactive, drivable browser viewport (CDP screencast + input). views: - { id: panel, label: "Browser", icon: Globe, path: /plugins/agent_browser/panel } diff --git a/pyproject.toml b/pyproject.toml index da4b09f..96af0ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "agent-browser-plugin" -version = "0.5.1" -description = "Browser-automation plugin for protoAgent, backed by vercel-labs/agent-browser (tools + skill + workflows + a live browser panel)." +version = "0.6.0" +description = "Browser-automation plugin for protoAgent, backed by vercel-labs/agent-browser (tools + skill + workflows + an interactive, drivable browser panel)." requires-python = ">=3.11" # Runtime deps come from the protoAgent host (langchain-core, fastapi). The only diff --git a/requirements-dev.txt b/requirements-dev.txt index a946ba1..a22a960 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,11 @@ # Test-only deps. At runtime the host (protoAgent) provides fastapi + langchain-core -# (and httpx/websockets for the dashboard proxy); the only runtime requirement is the -# `agent-browser` CLI on PATH. These let the host-free suite run standalone in CI. +# (and httpx/websockets for the interactive-panel CDP bridge); the only runtime +# requirement is the `agent-browser` CLI on PATH. These let the host-free suite run +# standalone in CI. fastapi>=0.110 langchain-core>=0.2 httpx>=0.27 +websockets>=12 pyyaml>=6 pytest>=8 pytest-asyncio>=0.23 diff --git a/requirements.txt b/requirements.txt index fd729e1..2f2bd34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -# No pip deps — the host (protoAgent) provides langchain-core + fastapi. The only -# external requirement is the `agent-browser` CLI on PATH: +# No pip deps — the host (protoAgent) provides langchain-core + fastapi, and `websockets` +# (a uvicorn[standard] extra) which the interactive panel uses to bridge Chrome's CDP. +# The only external requirement is the `agent-browser` CLI on PATH: # npm i -g agent-browser && agent-browser install diff --git a/tests/test_agent_browser.py b/tests/test_agent_browser.py index 3b782a1..ab1c517 100644 --- a/tests/test_agent_browser.py +++ b/tests/test_agent_browser.py @@ -1,15 +1,15 @@ """Tests for the agent_browser plugin — the tool subprocess wrappers (arg-building + -graceful error degradation), the panel routers (page / four-rules / gating / proxy), -register() wiring, the dashboard lifecycle, and manifest/version coherence. Host-free: -subprocess.run is mocked, so no agent-browser binary and no real browser are needed.""" +graceful error degradation), the interactive panel routes (page / ticket / WS gating / +nav), register() wiring, and manifest/version coherence. Host-free: subprocess.run is +mocked, so no agent-browser binary and no real browser are needed.""" from __future__ import annotations from pathlib import Path +import pytest import agent_browser.browser_panel as bp -import agent_browser.lifecycle as lc import agent_browser.tools as tools from conftest import fake_run @@ -51,18 +51,9 @@ async def test_action_tools_pass_refs(monkeypatch): assert rec[-1] == ["ab", "snapshot"] -async def test_dashboard_tool_validates_action_and_passes_port(monkeypatch): - rec = [] - monkeypatch.setattr(tools.subprocess, "run", fake_run(record=rec)) - t = _toolmap({"binary": "ab", "dashboard_port": 9000}) - assert "action must be" in await t["browser_dashboard"].ainvoke({"action": "boom"}) - await t["browser_dashboard"].ainvoke({"action": "start"}) - assert rec[-1] == ["ab", "dashboard", "start", "--port", "9000"] - - -def test_all_17_tools_present(): +def test_all_16_tools_present(): names = set(_toolmap()) - assert len(names) == 17 + assert len(names) == 16 assert { "browser_open", "browser_snapshot", @@ -71,8 +62,8 @@ def test_all_17_tools_present(): "browser_screenshot", "browser_eval", "browser_close", - "browser_dashboard", } <= names + assert "browser_dashboard" not in names # the dashboard tool is gone (full switchover) # ── the tools: graceful error degradation (a failed action informs, never crashes) ── @@ -105,15 +96,15 @@ async def test_nonzero_exit_surfaces_stderr(monkeypatch): # ── register() wiring ──────────────────────────────────────────────────────────── -def test_register_wires_tools_panel_data_and_surface(registry): +def test_register_wires_tools_and_panel_routers(registry): import agent_browser as pkg pkg.register(registry) - assert len(registry.tools) == 17 + assert len(registry.tools) == 16 prefixes = [p for p, _ in registry.routers] assert None in prefixes # the panel PAGE (host default prefix /plugins/agent_browser) assert "/api/plugins/agent_browser" in prefixes # gated data routes - assert registry.surfaces and registry.surfaces[0][0] == "agent-browser-dashboard" + assert registry.surfaces == [] # no dashboard lifecycle surface anymore # ── manifest / version coherence + settings ────────────────────────────────────── @@ -136,36 +127,15 @@ def test_settings_fields_are_valid_and_back_real_config(): m = yaml.safe_load((ROOT / "protoagent.plugin.yaml").read_text()) by_key = {f["key"]: f for f in m["settings"]} - assert by_key["panel_mode"]["type"] == "select" - assert set(by_key["panel_mode"]["options"]) == {"full", "minimal"} - assert by_key["headed"]["type"] == "bool" and by_key["dashboard_port"]["type"] == "number" + assert by_key["headed"]["type"] == "bool" and by_key["timeout_s"]["type"] == "number" + # the switchover dropped these knobs entirely: + assert "panel_mode" not in by_key and "dashboard_port" not in by_key + assert "panel_mode" not in m["config"] and "manage_dashboard" not in m["config"] # every settings key has a declared default in config: assert set(by_key) <= set(m["config"]) -# ── the dashboard lifecycle ────────────────────────────────────────────────────── - - -async def test_lifecycle_starts_on_boot_and_stops_on_shutdown(monkeypatch): - rec = [] - monkeypatch.setattr(lc.subprocess, "run", fake_run(record=rec)) - start, stop = lc.make_dashboard_surface({"binary": "ab", "dashboard_port": 9999}) - await start() - await stop() - assert ["ab", "dashboard", "start", "--port", "9999"] in rec - assert ["ab", "dashboard", "stop"] in rec - - -async def test_lifecycle_leaves_shared_dashboard_untouched(monkeypatch): - rec = [] - monkeypatch.setattr(lc.subprocess, "run", fake_run(record=rec)) - start, stop = lc.make_dashboard_surface({"manage_dashboard": False}) - await start() - await stop() - assert rec == [] # manage_dashboard:false → never touches the daemon - - -# ── the panel routers (page / four-rules / gating / proxy) ─────────────────────── +# ── the panel routes (page / ticket / WS gating / nav) ─────────────────────────── def _app(cfg=None): @@ -177,74 +147,64 @@ def _app(cfg=None): return app -def test_panel_page_full_mode_embeds_local_or_errors(): +def test_panel_page_wires_canvas_stream_and_input(): from fastapi.testclient import TestClient - c = TestClient(_app({"panel_mode": "full", "dashboard_port": 4955})) - r = c.get("/plugins/agent_browser/panel") - assert r.status_code == 200 - html = r.text + html = TestClient(_app({})).get("/plugins/agent_browser/panel").text assert "/_ds/plugin-kit.css" in html # DS kit assert 'location.pathname.split("/plugins/")[0]' in html # slug-aware base - assert 'id="f"' in html # the dashboard iframe - assert '"http://"+location.hostname+":"+PORT' in html # embed the dashboard's OWN local origin - assert "LOOPBACK" in html # local detection (loopback host + not fleet-proxied) - assert "Open the console locally" in html # the clear error shown when NOT local - assert "/api/plugins/agent_browser/dashboard" in html # the start/stop control - assert 'id="openlink"' in html and "Open ↗" in html # open the dashboard in a new tab - assert "panel_mode: minimal" in html # the remote alternative the error points at - # the dead sub-path proxy is gone; the port placeholder is interpolated. - assert "/panel/dash" not in html - assert "__DASH_PORT__" not in html and "4955" in html - - -def test_panel_page_minimal_mode_uses_gated_data_routes(): + assert 'id="cv"' in html and "createImageBitmap" in html # the canvas + frame painting + assert "/api/plugins/agent_browser/stream-ticket" in html # mint a ticket (gated) + assert "/api/plugins/agent_browser/stream" in html # the WS stream + assert 'u.protocol==="https:" ? "wss:" : "ws:"' in html # http→ws upgrade + assert 'send({t:"mouse"' in html and 'send({t:"key"' in html # input forwarding + assert "/api/plugins/agent_browser/nav" in html and "kit.apiFetch" in html # nav via gated route + # the removed dashboard-embed / screenshot modes leave no trace: + assert "/api/plugins/agent_browser/shot" not in html + assert 'id="f"' not in html and "Open the console locally" not in html + + +def test_stream_ticket_route_mints_a_ticket(): from fastapi.testclient import TestClient - html = TestClient(_app({"panel_mode": "minimal", "dashboard_port": 4933})).get( - "/plugins/agent_browser/panel" - ).text - assert "/api/plugins/agent_browser/shot" in html # gated screenshot - assert "/api/plugins/agent_browser/nav" in html # gated nav - assert "kit.apiFetch" in html # authed fetch via the kit - # the dashboard control (start it from the UI) + the port placeholder is interpolated - assert "/api/plugins/agent_browser/dashboard" in html and "Start dashboard" in html - assert "__DASH_PORT__" not in html and "4933" in html + body = TestClient(_app()).post("/api/plugins/agent_browser/stream-ticket").json() + assert isinstance(body.get("ticket"), str) and len(body["ticket"]) > 10 -def test_default_panel_mode_is_full(): - import yaml +def test_stream_ws_rejects_a_bad_ticket(): + from fastapi.testclient import TestClient + from starlette.websockets import WebSocketDisconnect - m = yaml.safe_load((ROOT / "protoagent.plugin.yaml").read_text()) - assert m["config"]["panel_mode"] == "full" # embed the dashboard by default (local setup) - by_key = {f["key"]: f for f in m["settings"]} - assert by_key["panel_mode"]["options"][0] == "full" # default listed first + c = TestClient(_app()) + # no valid ticket → handler closes (1008) before accept → connect raises. + with pytest.raises(WebSocketDisconnect): + with c.websocket_connect("/api/plugins/agent_browser/stream?ticket=nope"): + pass -def test_dashboard_control_endpoint(monkeypatch): +def test_stream_ws_accepts_valid_ticket_then_reports_no_page(monkeypatch): from fastapi.testclient import TestClient - c = TestClient(_app({"dashboard_port": 4934})) # nothing listens → status "stopped" - st = c.get("/api/plugins/agent_browser/dashboard").json() - assert st["running"] is False and st["port"] == 4934 - # start/stop run the CLI (mocked); bad action is rejected. - rec = [] - monkeypatch.setattr(bp.subprocess, "run", fake_run(record=rec)) - assert c.post("/api/plugins/agent_browser/dashboard", json={"action": "x"}).json()["ok"] is False - assert c.post("/api/plugins/agent_browser/dashboard", json={"action": "start"}).json()["ok"] is True - assert ["agent-browser", "dashboard", "start", "--port", "4934"] in rec - assert c.post("/api/plugins/agent_browser/dashboard", json={"action": "stop"}).json()["ok"] is True - assert ["agent-browser", "dashboard", "stop"] in rec + # resolve returns no page → the handler accepts, sends an error frame, and closes + # (exercises the ticket gate + accept path without a real browser/CDP). + monkeypatch.setattr(bp.browser_stream, "resolve_page_target", + lambda binary, timeout: (None, "no page open")) + c = TestClient(_app()) + ticket = c.post("/api/plugins/agent_browser/stream-ticket").json()["ticket"] + with c.websocket_connect(f"/api/plugins/agent_browser/stream?ticket={ticket}") as ws: + assert ws.receive_json() == {"t": "error", "msg": "no page open"} -def test_shot_route_503s_without_a_frame(monkeypatch): +def test_stream_ticket_is_single_use(): from fastapi.testclient import TestClient + from starlette.websockets import WebSocketDisconnect - monkeypatch.setattr(bp.subprocess, "run", fake_run(rc=127, stderr="not found")) - monkeypatch.setattr(bp.os.path, "exists", lambda p: False) # no cached frame - monkeypatch.setattr(bp, "_shot_ts", 0.0) - r = TestClient(_app()).get("/api/plugins/agent_browser/shot") - assert r.status_code == 503 + c = TestClient(_app()) + ticket = c.post("/api/plugins/agent_browser/stream-ticket").json()["ticket"] + assert bp.browser_stream.consume_ticket(ticket) is True # burn it directly + with pytest.raises(WebSocketDisconnect): # replay is rejected + with c.websocket_connect(f"/api/plugins/agent_browser/stream?ticket={ticket}"): + pass def test_nav_route_validates(monkeypatch): diff --git a/tests/test_browser_stream.py b/tests/test_browser_stream.py new file mode 100644 index 0000000..8d6ff00 --- /dev/null +++ b/tests/test_browser_stream.py @@ -0,0 +1,166 @@ +"""Tests for the interactive-panel CDP bridge — the pure, host-free brains of +``browser_stream``: CDP endpoint parsing, page-target selection, and the +input-message → CDP-command translation. These lock in the behavior validated +end-to-end against a live browser (mouse + keyboard land; screencast frames flow). +No websockets/binary/browser needed — the pure functions have no IO.""" + +from __future__ import annotations + +import agent_browser.browser_stream as bs + + +# ── CDP endpoint parsing ───────────────────────────────────────────────────────── + + +def test_http_base_from_ws_url(): + assert bs._http_base_from_ws( + "ws://127.0.0.1:52886/devtools/browser/4580bed9") == "http://127.0.0.1:52886" + + +# ── page-target selection ──────────────────────────────────────────────────────── + + +def _t(type_, url, ws="ws://x"): + return {"type": type_, "url": url, "webSocketDebuggerUrl": ws} + + +def test_pick_page_prefers_current_url(): + targets = [ + _t("page", "https://a.com", "ws://a"), + _t("page", "https://b.com", "ws://b"), + ] + assert bs.pick_page_target(targets, "https://b.com") == "ws://b" + + +def test_pick_page_falls_back_to_first_real_page(): + targets = [ + _t("page", "chrome://newtab/", "ws://newtab"), + _t("page", "https://real.com", "ws://real"), + ] + # no current-url match → skip chrome:// surfaces, take the first real page. + assert bs.pick_page_target(targets, "https://gone.com") == "ws://real" + + +def test_pick_page_skips_non_page_and_missing_ws(): + targets = [ + {"type": "service_worker", "url": "x", "webSocketDebuggerUrl": "ws://sw"}, + {"type": "page", "url": "https://c.com"}, # no ws url → not streamable + _t("page", "https://d.com", "ws://d"), + ] + assert bs.pick_page_target(targets, "") == "ws://d" + + +def test_pick_page_none_when_no_pages(): + assert bs.pick_page_target([{"type": "iframe", "url": "x", "webSocketDebuggerUrl": "ws://i"}], "") is None + assert bs.pick_page_target([], "") is None + + +# ── input translation: mouse ───────────────────────────────────────────────────── + + +def test_mouse_down_maps_to_pressed_with_button_and_count(): + method, p = bs.input_to_cdp( + {"t": "mouse", "action": "down", "x": 10, "y": 20, "button": "left", "clickCount": 2, "buttons": 1}) + assert method == "Input.dispatchMouseEvent" + assert p["type"] == "mousePressed" and p["x"] == 10.0 and p["y"] == 20.0 + assert p["button"] == "left" and p["clickCount"] == 2 and p["buttons"] == 1 + + +def test_mouse_move_has_no_button_or_clickcount(): + _, p = bs.input_to_cdp({"t": "mouse", "action": "move", "x": 5, "y": 6}) + assert p["type"] == "mouseMoved" + assert "button" not in p and "clickCount" not in p + + +def test_mouse_up_maps_to_released(): + _, p = bs.input_to_cdp({"t": "mouse", "action": "up", "x": 1, "y": 2, "button": "left"}) + assert p["type"] == "mouseReleased" + + +def test_wheel_maps_to_mousewheel_deltas(): + method, p = bs.input_to_cdp({"t": "wheel", "x": 3, "y": 4, "dx": 0, "dy": 120}) + assert method == "Input.dispatchMouseEvent" and p["type"] == "mouseWheel" + assert p["deltaX"] == 0.0 and p["deltaY"] == 120.0 + + +# ── input translation: keyboard ────────────────────────────────────────────────── + + +def test_key_down_with_text_carries_text_and_vkey(): + method, p = bs.input_to_cdp( + {"t": "key", "action": "down", "key": "a", "code": "KeyA", "text": "a", "keyCode": 65}) + assert method == "Input.dispatchKeyEvent" and p["type"] == "keyDown" + assert p["text"] == "a" and p["key"] == "a" and p["code"] == "KeyA" + assert p["windowsVirtualKeyCode"] == 65 and p["nativeVirtualKeyCode"] == 65 + + +def test_key_up_omits_text(): + _, p = bs.input_to_cdp({"t": "key", "action": "up", "key": "a", "code": "KeyA", "text": "a"}) + assert p["type"] == "keyUp" and "text" not in p + + +def test_modifiers_bitmask(): + _, p = bs.input_to_cdp({"t": "mouse", "action": "move", "x": 0, "y": 0, "ctrl": True, "shift": True}) + assert p["modifiers"] == (2 | 8) # Ctrl=2, Shift=8 + + +# ── input translation: rejects non-drivable messages ───────────────────────────── + + +def test_unknown_messages_return_none(): + assert bs.input_to_cdp({"t": "bogus"}) is None + assert bs.input_to_cdp({"t": "mouse", "action": "wat"}) is None + assert bs.input_to_cdp({"t": "key", "action": "wat"}) is None + + +# ── resolve_page_target: IO paths (subprocess + json/list mocked) ───────────────── + + +def test_resolve_returns_note_when_no_session(monkeypatch): + def no_url(args, **kw): + import types + return types.SimpleNamespace(returncode=1, stdout="", stderr="no session") + monkeypatch.setattr(bs.subprocess, "run", no_url) + ws, note = bs.resolve_page_target("ab") + assert ws is None and "no session" in note + + +def test_resolve_finds_active_page(monkeypatch): + import io + import types + + def fake_run(args, **kw): + if args[1:] == ["get", "cdp-url"]: + return types.SimpleNamespace(returncode=0, stdout="ws://127.0.0.1:9/devtools/browser/x\n", stderr="") + if args[1:] == ["get", "url"]: + return types.SimpleNamespace(returncode=0, stdout="https://ex.com\n", stderr="") + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(bs.subprocess, "run", fake_run) + listing = b'[{"type":"page","url":"https://ex.com","webSocketDebuggerUrl":"ws://127.0.0.1:9/devtools/page/P"}]' + monkeypatch.setattr(bs.urllib.request, "urlopen", + lambda url, timeout=0: io.BytesIO(listing)) + ws, note = bs.resolve_page_target("ab") + assert ws == "ws://127.0.0.1:9/devtools/page/P" and note == "" + + +# ── WS auth tickets: single-use + expiry ───────────────────────────────────────── + + +def test_ticket_mint_then_consume_is_single_use(): + t = bs.mint_ticket() + assert bs.consume_ticket(t) is True # first use validates + assert bs.consume_ticket(t) is False # replay is burned + + +def test_consume_rejects_unknown_and_empty(): + assert bs.consume_ticket("never-minted") is False + assert bs.consume_ticket("") is False + + +def test_ticket_expires_after_ttl(monkeypatch): + clock = {"t": 1000.0} + monkeypatch.setattr(bs.time, "monotonic", lambda: clock["t"]) + t = bs.mint_ticket() + clock["t"] += bs._TICKET_TTL + 1 # advance past the TTL + assert bs.consume_ticket(t) is False diff --git a/tools.py b/tools.py index cab2835..ef98593 100644 --- a/tools.py +++ b/tools.py @@ -25,7 +25,6 @@ def get_browser_tools(cfg: dict | None): cfg = cfg or {} binary = str(cfg.get("binary") or "agent-browser") timeout = float(cfg.get("timeout_s", 60)) - port = int(cfg.get("dashboard_port", 4848)) def _launch_flags() -> list[str]: """Curated runtime knobs → agent-browser global flags, applied when the @@ -146,7 +145,7 @@ async def browser_eval(expression: str) -> str: Use sparingly — prefer snapshot + the action tools.""" return await _ab("eval", expression) - # ── capture + lifecycle ─────────────────────────────────────────────────── + # ── capture + session ───────────────────────────────────────────────────── @tool async def browser_screenshot(path: str = "page.png") -> str: """Save a screenshot of the current page to `path`. Returns the file path.""" @@ -157,20 +156,9 @@ async def browser_close() -> str: """Close the browser session. Call when the task is done to free the daemon.""" return await _ab("close") - @tool - async def browser_dashboard(action: str = "start") -> str: - """Manage the live observability dashboard (the Browser panel embeds it). - `action` is `start` (background, port from config), `stop`, or `status`.""" - act = (action or "start").strip().lower() - if act not in ("start", "stop", "status"): - return f"Error: action must be start|stop|status, got {action!r}" - if act == "start": - return await _ab("dashboard", "start", "--port", str(port)) - return await _ab("dashboard", act) - return [ browser_open, browser_back, browser_forward, browser_reload, browser_snapshot, browser_get_text, browser_get_html, browser_get_value, browser_click, browser_fill, browser_type, browser_press, browser_hover, - browser_eval, browser_screenshot, browser_close, browser_dashboard, + browser_eval, browser_screenshot, browser_close, ] From bb4cda07c70cd484e7a9d04961da8d8ff1cade57 Mon Sep 17 00:00:00 2001 From: Josh Mabry Date: Wed, 1 Jul 2026 00:57:42 -0700 Subject: [PATCH 2/2] feat(panel): Start button + configurable homepage (home_url) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty viewport was a dead end. Now, when no page is open, the panel shows a Start button, and a new `home_url` config sets the page it opens to: - home_url set → the panel auto-opens it once; the button reads "Open " - home_url blank → the button opens about:blank (no auto-open) home_url is injected as a JS string literal via json.dumps + `<`→< so a value containing `` can't break out of the inline script. Editable in Settings ▸ Plugins. 35 tests pass; ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ README.md | 3 +++ browser_panel.py | 28 +++++++++++++++++++++++++--- protoagent.plugin.yaml | 4 ++++ tests/test_agent_browser.py | 14 ++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 316ef2a..fa0fe21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ - **How it works.** A second CDP client attaches to the same Chrome agent-browser drives (via `agent-browser get cdp-url`); `browser_stream.py` bridges `Page.startScreencast` ⇆ the panel. The nav toolbar (url / back / forward / reload) reuses the gated HTTP `/nav` route. +- **Start button + configurable homepage.** When no page is open, the panel shows a **Start + button** instead of a dead end. A new `home_url` config sets the page it opens to: when set, + the panel **auto-opens** it (and the button reads “Open ”); blank → the button opens + `about:blank`. Editable in Settings ▸ Plugins. - **WebSocket auth.** The host's operator-bearer gate is HTTP-only and does **not** cover WS handshakes, so the stream self-gates: the panel mints a **single-use ticket** from the gated `POST /stream-ticket` (bearer-checked) and presents it on the WS URL; the handler validates and diff --git a/README.md b/README.md index dba7d2e..3f093db 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,9 @@ point. **host and a remote fleet member** alike. A second CDP client attaches to the same Chrome agent-browser drives (`agent-browser get cdp-url`); `browser_stream.py` does the bridging. + When no page is open the panel shows a **Start button** (not a dead end). Set `home_url` to a + page and the panel **auto-opens** it — a homepage — otherwise Start opens `about:blank`. + **WebSocket auth:** the host's operator-bearer gate is HTTP-only and doesn't cover WS handshakes, so the stream self-gates — the panel mints a **single-use ticket** from the gated `POST /api/plugins/agent_browser/stream-ticket` and presents it on the WS URL. diff --git a/browser_panel.py b/browser_panel.py index f03e02b..0435e83 100644 --- a/browser_panel.py +++ b/browser_panel.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import json import logging import subprocess @@ -29,11 +30,18 @@ def build_panel_router(cfg: dict | None): + cfg = cfg or {} + home = str(cfg.get("home_url") or "").strip() router = APIRouter() + # A safe JS string literal for `const HOME=__HOME_URL__;`. json.dumps handles quote/ + # backslash escaping; the `<` → < step stops a `` in the value from + # closing the inline script tag in the HTML parser (still the same string in JS). + home_literal = json.dumps(home).replace("<", "\\u003c") + @router.get("/panel") async def _panel(): - return HTMLResponse(_INTERACTIVE_PAGE) + return HTMLResponse(_INTERACTIVE_PAGE.replace("__HOME_URL__", home_literal)) return router @@ -181,6 +189,7 @@ async def _nav(body: dict = Body(...)): catch (e) { kit = { initPluginView(cb){ if(cb) cb(); }, apiFetch:(p,i)=>fetch(BASE+p,i), apiUrl:(p)=>BASE+p }; } const $=(id)=>document.getElementById(id); const cv=$("cv"), ctx=cv.getContext("2d"); +const HOME=__HOME_URL__; // configured homepage (blank ⇒ Start opens about:blank, no auto-open) // ── nav toolbar — reuses the gated HTTP /nav route (agent-browser open/back/…) ── async function nav(action,url){ @@ -196,6 +205,20 @@ async def _nav(body: dict = Body(...)): function showMsg(t,html){ $("mt").textContent=t; $("md").innerHTML=html||""; $("msg").style.display="flex"; } function hideMsg(){ $("msg").style.display="none"; } +// ── the empty state: a Start button (+ a one-shot auto-open of the homepage) ─── +let autoStarted=false; +function homeHost(){ try{ return new URL(HOME).host || HOME; }catch(_){ return HOME; } } +function startBrowser(){ nav("open", HOME || "about:blank"); } +window.startBrowser=startBrowser; +function showStart(note){ + const label = HOME ? ("Open " + homeHost()) : "Start browser"; + showMsg("No page open", + (note ? note + "

" : "") + + '' + + '
or type a URL above, or let the agent drive — browser_open.
'); + if(HOME && !autoStarted){ autoStarted=true; startBrowser(); } // open the configured homepage once +} + // ── the interactive stream: mint a ticket (gated) → open the WS → paint frames ── let ws=null, connected=false, devW=1280, devH=800, retry=null; function wsUrl(ticket){ @@ -222,8 +245,7 @@ async def _nav(body: dict = Body(...)): if(typeof ev.data==="string"){ let m; try{ m=JSON.parse(ev.data); }catch(_){ return; } if(m.t==="meta"){ if(m.w) devW=m.w; if(m.h) devH=m.h; } - else if(m.t==="error"){ setStatus("err","no page"); - showMsg("Nothing to show yet",(m.msg||"")+"

Type a URL above, or let the agent drive — browser_open."); } + else if(m.t==="error"){ setStatus("err","no page"); showStart(m.msg); } return; } try{ // binary → a JPEG screencast frame diff --git a/protoagent.plugin.yaml b/protoagent.plugin.yaml index 84196e7..5281822 100644 --- a/protoagent.plugin.yaml +++ b/protoagent.plugin.yaml @@ -23,6 +23,9 @@ config_section: agent_browser config: binary: agent-browser # the agent-browser CLI on PATH (override for a pinned path) timeout_s: 60 # per-command subprocess timeout + home_url: "" # homepage the panel opens to. When set, the panel auto-opens it if + # no browser page is open; the empty state also shows a Start button. + # Blank → the Start button opens about:blank (no auto-open). # Runtime / launch options — a curated set passed to `agent-browser open` so you can # shape and lock down the browser the agent spins up. Blank/0/false = CLI default. headed: false # show a real browser window instead of headless @@ -34,6 +37,7 @@ config: # Editable in Settings ▸ Plugins (ADR 0019) — the operator knobs above as UI fields. settings: + - { key: home_url, label: "Homepage URL", type: string, description: "The page the interactive panel opens to. When set, it auto-opens if no browser page is open, and the empty state shows a Start button. Blank = a Start button that opens about:blank." } - { key: headed, label: "Headed browser", type: bool, description: "Show a real browser window instead of running headless." } - { key: allowed_domains, label: "Allowed domains", type: string, description: "Comma-separated navigable-domain allowlist (e.g. example.com,*.foo.com). Blank = unrestricted." } - { key: confirm_actions, label: "Confirm actions", type: string, description: "Comma-separated action categories that require confirmation before the agent runs them." } diff --git a/tests/test_agent_browser.py b/tests/test_agent_browser.py index ab1c517..ccd7d1b 100644 --- a/tests/test_agent_browser.py +++ b/tests/test_agent_browser.py @@ -159,11 +159,25 @@ def test_panel_page_wires_canvas_stream_and_input(): assert 'u.protocol==="https:" ? "wss:" : "ws:"' in html # http→ws upgrade assert 'send({t:"mouse"' in html and 'send({t:"key"' in html # input forwarding assert "/api/plugins/agent_browser/nav" in html and "kit.apiFetch" in html # nav via gated route + assert "startBrowser" in html and 'const HOME="";' in html # empty-state Start; blank home default # the removed dashboard-embed / screenshot modes leave no trace: assert "/api/plugins/agent_browser/shot" not in html assert 'id="f"' not in html and "Open the console locally" not in html +def test_panel_home_url_is_injected_safely(): + from fastapi.testclient import TestClient + + # a configured homepage lands as a JS string literal the Start button + auto-open use + html = TestClient(_app({"home_url": "https://example.com"})).get("/plugins/agent_browser/panel").text + assert 'const HOME="https://example.com";' in html + assert "__HOME_URL__" not in html # placeholder fully interpolated + # a -injection attempt is escaped: the quote is JSON-escaped and the `<` + # becomes <, so it neither breaks the JS string nor closes the inline script. + evil = TestClient(_app({"home_url": '"'})).get("/plugins/agent_browser/panel").text + assert 'const HOME="\\"\\u003c/script>";' in evil + + def test_stream_ticket_route_mints_a_ticket(): from fastapi.testclient import TestClient