diff --git a/control-plane/browse.py b/control-plane/browse.py index 124f2ac..93ff3dd 100644 --- a/control-plane/browse.py +++ b/control-plane/browse.py @@ -11,6 +11,8 @@ click_element re-derive the same numbered list, verify the ref still matches, scroll it into view, then fire a real OS-level click through /action at the computed screen coordinates (isTrusted stays true) + — and return the settled page with it, so the caller never has to + spend a second LLM turn asking what the click did fill batch-fill a form via native value setters + input/change events (React-safe); refuses password fields — vault login owns those wait_for server-side 0.4s poll for selector/text/network-idle, so agents @@ -22,6 +24,13 @@ emits elements in document order, so a ref from the last snapshot re-derives to the same element unless the page itself changed — in which case click_element refuses and returns a fresh snapshot instead of clicking the wrong thing. + +A ref is that document-order index, which is why snapshot can show any subset it +likes without breaking anything. It shows the on-screen ones first: a big page has +thousands of interactive elements and the cap has to fall somewhere, and falling on +"whoever appears first in the HTML" spends it on chrome the reader scrolled past. +Measured on a scrolled Wikipedia article: a third of what was on screen missed the +cut. The shown numbers therefore skip. """ import json import re @@ -34,8 +43,9 @@ MAX_ELS = 150 -# The shared walk. Defines __els = [{el, tag, type, name, value, href}] in -# document order, visible interactive elements only. Password values never read. +# The shared walk. Defines __els = [{el, tag, type, name, value, href, on}] in +# document order, visible interactive elements only (`on` = 0 when the element is in +# the viewport right now). Password values never read. _WALK = """ const __sel='a[href],button,input,select,textarea,summary,label,'+ '[role="button"],[role="link"],[role="tab"],[role="menuitem"],[role="checkbox"],'+ @@ -55,7 +65,8 @@ el.innerText||el.title||el.alt||'').trim().replace(/\\s+/g,' ').slice(0,80); __els.push({el,tag,type:(el.type||el.getAttribute('role')||''),name, value:('value'in el&&el.type!=='password'&&tag!=='button')?String(el.value).slice(0,40):'', - href:tag==='a'?String(el.getAttribute('href')||'').slice(0,120):''}); + href:tag==='a'?String(el.getAttribute('href')||'').slice(0,120):'', + on:(r.bottom>0&&r.top0&&r.leftr)};" % MAX_ELS) + """The active tab's interactive elements as numbered lines, everything on screen + first. Refs are indices into the whole walk, so the shown list can skip numbers.""" + body = (""" +const __by=__els.map((e,i)=>i); +__by.sort((a,b)=>__els[a].on-__els[b].on||a-b); // on screen first, document order within +const __pick=__by.slice(0,%d).sort((a,b)=>a-b); // then hand them back in reading order +return {url:location.href,title:document.title,count:__els.length, + els:__pick.map(i=>{const{el,on,...r}=__els[i];return{i,...r};})}; +""" % MAX_ELS) r = eval_js(row, _iife(body), timeout_s) if not r.get("ok"): return {"ok": False, "error": r.get("error") or "snapshot failed"} @@ -89,7 +106,7 @@ def snapshot(row, timeout_s=15): return {"ok": True, "url": v.get("url"), "title": v.get("title"), "count": v.get("count", len(els)), "truncated": (v.get("count", 0) > MAX_ELS), - "elements": [_fmt(i, e) for i, e in enumerate(els)]} + "elements": [_fmt(e.get("i", i), e) for i, e in enumerate(els)]} def _locate(row, ref, name, timeout_s=15): @@ -113,10 +130,96 @@ def _locate(row, ref, name, timeout_s=15): return r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad locate result"} -def click_element(row, ref, name=None, text=None, screenshot=False): - """Verify + scroll + real OS click. On stale ref: refuse and hand back a fresh - snapshot (a wrong click is worse than a slow click). text, when given, is typed - after the click (the click focuses the field).""" +# Same sentinel trick deskclient.navigate uses, and for the same reason: a click that +# navigates leaves the OLD document reporting readyState 'complete' for a beat, so +# polling readyState alone snapshots the page the agent just left. Stamping the +# document before the action turns that race into a check — a document without the +# stamp is, by construction, the one the action produced. +_STAMP = "window.__case_act" + + +def _stamp(row): + try: + return bool(eval_js(row, f"{_STAMP}=1", 3).get("ok")) + except ApiError: + return False + + +def _settled_snapshot(row, stamped, settle_s=1.0, budget_s=8.0): + """The page as it stands once the action stops moving it. + + Two endings, and they need telling apart. If the stamp is gone the action + navigated: wait for the new document to finish and snapshot that. If the stamp is + still there after `settle_s`, nothing navigated and this page IS the answer — a + grace that long because a JS-driven navigation does not commit instantly. A 502 + means the context was torn down mid-navigation, which is progress, so poll through. + + If the stamp never landed, `__case_act===undefined` is the starting document, + not proof of navigation. Wait out the grace and snapshot whatever is complete. + + This costs ~2 extra eval round trips at ~11ms each. The computer_snapshot call it + saves the agent is a whole model turn. Cheap side of a very lopsided trade. + """ + if not stamped: + time.sleep(settle_s) + deadline = time.time() + budget_s + while time.time() < deadline: + time.sleep(0.25) + try: + r = eval_js(row, "document.readyState", 3) + except ApiError as e: + if e.status != 502: + return None + continue + if r.get("value") == "complete": + break + try: + return snapshot(row) + except ApiError: + return None + deadline = time.time() + budget_s + grace = time.time() + settle_s + while time.time() < deadline: + time.sleep(0.25) + try: + r = eval_js(row, f"[{_STAMP}===undefined,document.readyState]", 3) + except ApiError as e: + if e.status != 502: + return None + continue # context gone: the navigation we are waiting for + v = r.get("value") + if not isinstance(v, list): + break + navigated, ready = v[0], v[1] + if navigated and ready == "complete": + return snapshot(row) + if not navigated and time.time() >= grace: + fresh = snapshot(row) + # don't leave a uniquely-named global on a live page for site JS to find + try: + eval_js(row, f"delete {_STAMP}", 3) + except ApiError: + pass + return fresh + try: + return snapshot(row) # never settled; a partial list still beats a blind turn + except ApiError: + return None + + +def _attach_snapshot(row, res, want, stamped): + if not want: + return res + fresh = _settled_snapshot(row, stamped) + if fresh and fresh.get("ok"): + res["snapshot"] = fresh + return res + + +def click_element(row, ref, name=None, text=None, screenshot=False, snapshot_after=True): + """Verify + scroll + real OS click, then hand back the page it produced. On stale + ref: refuse and hand back a fresh snapshot (a wrong click is worse than a slow + click). text, when given, is typed after the click (the click focuses the field).""" loc = _locate(row, ref, name) if not loc.get("ok"): if loc.get("stale"): @@ -129,6 +232,7 @@ def click_element(row, ref, name=None, text=None, screenshot=False): x, y = loc["x"], loc["y"] if not (0 <= x < SCREEN_W and 0 <= y < SCREEN_H): return {"ok": False, "error": f"element resolves off-screen ({x},{y})"} + stamped = snapshot_after and _stamp(row) out = desk_json(row, "POST", "/action", json={"type": "click", "x": x, "y": y, "screenshot": bool(screenshot) and not text}, @@ -141,10 +245,10 @@ def click_element(row, ref, name=None, text=None, screenshot=False): res = {"ok": True, "clicked": loc.get("name"), "tag": loc.get("tag"), "x": x, "y": y} if isinstance(out, dict) and out.get("screenshot_png_b64"): res["screenshot_png_b64"] = out["screenshot_png_b64"] - return res + return _attach_snapshot(row, res, snapshot_after, stamped) -def fill(row, fields, submit=False, timeout_s=20): +def fill(row, fields, submit=False, timeout_s=20, snapshot_after=True): """Batch form fill. fields=[{ref, value}]. Native setters + input/change events so React/Vue see the change. Password inputs are refused inside the page — vaulted computer_login owns credentials, always.""" @@ -182,7 +286,7 @@ def fill(row, fields, submit=False, timeout_s=20): out.push({ref:f.ref,ok:true,name:e.name}); }catch(err){out.push({ref:f.ref,ok:false,error:String(err).slice(0,80)});} } -if(__submit&&out.some(o=>o.ok)){ +if(__submit&&out.length&&out.every(o=>o.ok)){ const first=__els[__fields[0].ref]; const form=first&&first.el.form; if(form){form.requestSubmit?form.requestSubmit():form.submit();} @@ -190,10 +294,15 @@ def fill(row, fields, submit=False, timeout_s=20): } return {ok:out.every(o=>o.ok),fields:out}; """ + # Only the submit moves the page out from under the caller; after a plain fill the + # refs they already hold are still good, so don't spend the settle on one. + want = bool(snapshot_after and submit) + stamped = want and _stamp(row) r = eval_js(row, _iife(body), timeout_s) if not r.get("ok"): return {"ok": False, "error": r.get("error") or "fill failed"} - return r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad fill result"} + out = r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad fill result"} + return _attach_snapshot(row, out, want and out.get("ok"), stamped) def wait_for(row, selector=None, text=None, gone=False, network_idle=False, timeout_s=30): diff --git a/control-plane/cased.py b/control-plane/cased.py index 3e1061c..48282bb 100644 --- a/control-plane/cased.py +++ b/control-plane/cased.py @@ -354,7 +354,12 @@ def navigate_(cid: str, body: dict = Body(...), wake: bool = False): if "url" not in body: raise ApiError(400, "bad_request", "body needs 'url'") timeout = max(1, min(int(body.get("timeout_s") or 30), 120)) # never navigate then - return navigate(row, body["url"], timeout) # report failure at t=0 + out = navigate(row, body["url"], timeout) # report failure at t=0 + if out.get("ok") and body.get("snapshot", True): + fresh = browse.snapshot(row) # navigate already waited for readyState + if fresh.get("ok"): + out["snapshot"] = fresh + return out # ---------- element-level browsing (browse.py; control-plane composition) ---------- @@ -372,13 +377,15 @@ def click_(cid: str, body: dict = Body(...), wake: bool = False): raise ApiError(400, "bad_request", "body needs 'ref' (from GET /page)") return browse.click_element(row, int(body["ref"]), name=body.get("name"), text=body.get("text"), - screenshot=bool(body.get("screenshot"))) + screenshot=bool(body.get("screenshot")), + snapshot_after=bool(body.get("snapshot", True))) @app.post("/v1/computers/{cid}/fill") def fill_(cid: str, body: dict = Body(...), wake: bool = False): with awake(cid, wake) as row: - return browse.fill(row, body.get("fields"), submit=bool(body.get("submit"))) + return browse.fill(row, body.get("fields"), submit=bool(body.get("submit")), + snapshot_after=bool(body.get("snapshot", True))) @app.post("/v1/computers/{cid}/wait") diff --git a/control-plane/lifecycle.py b/control-plane/lifecycle.py index 5262c32..f48f523 100644 --- a/control-plane/lifecycle.py +++ b/control-plane/lifecycle.py @@ -9,6 +9,7 @@ transitioning it, so it deliberately bypasses the transition guard. """ import secrets +import time from config import DESK_H, DESK_W, IMAGE, MAX_RAM_MB, MAX_RUNNING, log from errors import ApiError @@ -210,6 +211,7 @@ def do_wake(cid): row = get_computer(cid) if row["state"] == "running" and dockerd.container_up(cid): return # DB can lie after a daemon restart, only skip if the container really is up + t0 = time.monotonic() # Asleep computers don't count against the budget; waking one must, same as create. # (create already checks; wake used to bypass the cap and OOM a small box.) if row["state"] == "asleep": @@ -225,7 +227,12 @@ def do_wake(cid): row["desk_token"]) desk_port, vnc_port = dockerd.container_ports(dockerd.get_container(cid)) store.set_ports(cid, desk_port, vnc_port) + t_container = time.monotonic() - t0 deskclient.wait_desk(cid, desk_port, row["desk_token"], 30) + # The two halves of a wake bill very differently — container start is docker, + # the rest is Chromium coming up. Split them or you tune the wrong one. + log.info("wake %s: container %.2fs, deskd healthy %.2fs", + cid, t_container, time.monotonic() - t0) except Exception: _try_set(cid, "asleep") # tolerate a concurrent delete here too raise diff --git a/image/Dockerfile b/image/Dockerfile index 5696737..64cd5c0 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -27,9 +27,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # at startup. Invalid dummy keys keep those services off without the "API keys # missing" infobar. Web logins (gmail.com etc.) are cookie logins, unaffected. # Policies disable the service machinery itself; matching kill flags in start.sh. +# MemorySaverModeSavings (1 = balanced) rides along here because it is a policy too: +# desks are long-lived and now restore their tabs across wakes, so background tabs +# accumulate. Discarded tabs reload from cookies on return, which agents tolerate. RUN printf 'export GOOGLE_API_KEY="no"\nexport GOOGLE_DEFAULT_CLIENT_ID="no"\nexport GOOGLE_DEFAULT_CLIENT_SECRET="no"\n' > /etc/chromium.d/apikeys \ && mkdir -p /etc/chromium/policies/managed \ - && printf '%s' '{"BrowserSignin":0,"SyncDisabled":true,"MetricsReportingEnabled":false,"SafeBrowsingProtectionLevel":0,"BackgroundModeEnabled":false,"ComponentUpdatesEnabled":false,"PromotionalTabsEnabled":false,"UrlKeyedAnonymizedDataCollectionEnabled":false,"DefaultSearchProviderEnabled":true,"DefaultSearchProviderName":"DuckDuckGo","DefaultSearchProviderKeyword":"duckduckgo.com","DefaultSearchProviderSearchURL":"https://duckduckgo.com/?q={searchTerms}","DefaultSearchProviderSuggestURL":"https://duckduckgo.com/ac/?q={searchTerms}&type=list"}' \ + && printf '%s' '{"BrowserSignin":0,"SyncDisabled":true,"MetricsReportingEnabled":false,"SafeBrowsingProtectionLevel":0,"BackgroundModeEnabled":false,"ComponentUpdatesEnabled":false,"PromotionalTabsEnabled":false,"UrlKeyedAnonymizedDataCollectionEnabled":false,"MemorySaverModeSavings":1,"DefaultSearchProviderEnabled":true,"DefaultSearchProviderName":"DuckDuckGo","DefaultSearchProviderKeyword":"duckduckgo.com","DefaultSearchProviderSearchURL":"https://duckduckgo.com/?q={searchTerms}","DefaultSearchProviderSuggestURL":"https://duckduckgo.com/ac/?q={searchTerms}&type=list"}' \ > /etc/chromium/policies/managed/case.json ARG NOVNC_VERSION=1.5.0 diff --git a/image/start.sh b/image/start.sh index f520dba..b51bdd6 100644 --- a/image/start.sh +++ b/image/start.sh @@ -4,14 +4,31 @@ # Traps SIGTERM (docker stop = sleep) so Chromium exits cleanly and flushes its profile. RES="${DESK_RESOLUTION:-1280x800x24}" +# Xvfb needs WxHxD and dies on bare WxH; depth 24 is the only one deskd's +# XWD->PNG grab supports (32bpp), so it is also the only default we append. +case "$RES" in *x*x*) ;; *) RES="${RES}x24" ;; esac # libXcursor honors this in every X client — the one switch that themes the # cursor everywhere (chromium reads it via GTK settings too, seeded below). export XCURSOR_THEME=case XCURSOR_SIZE=32 -# Seed per-user desktop config when missing (first boot, or volumes from older -# images); never overwrite — the volume is the user's. -seed() { [ -f "$2" ] || { mkdir -p "$(dirname "$2")"; cp "$1" "$2"; }; } +# Seed per-user desktop config from /usr/share/case. Two rules fight here: never +# stomp the user's own tweaks, but a shipped look change has to actually land. +# LOOK settles it. The volume outlives the image (it IS the computer's identity), +# so a desk created before a look change keeps writing back its old xfconf files +# and a missing-only seed silently skips every one of them — leaving the new +# wallpaper/panel/dock sitting unused in the image. Bump LOOK whenever the +# assets change: every desk force-seeds exactly once (resetting desktop tweaks +# that one time), then goes back to leaving the user alone. +LOOK=v2 +STAMP=~/.config/.case-look +seeded=1 +if [ "$(cat "$STAMP" 2>/dev/null)" = "$LOOK" ]; then + seed() { [ -f "$2" ] || { mkdir -p "$(dirname "$2")" && cp "$1" "$2"; } || seeded=0; } +else + echo "[start] look $LOOK — re-seeding desktop config" >&2 + seed() { mkdir -p "$(dirname "$2")" && cp -f "$1" "$2" || seeded=0; } +fi seed /usr/share/case/gtk-settings.ini ~/.config/gtk-3.0/settings.ini seed /usr/share/case/xfce4-panel.xml ~/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml seed /usr/share/case/xfwm4.xml ~/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml @@ -20,6 +37,9 @@ seed /usr/share/case/xfce4-desktop.xml ~/.config/xfce4/xfconf/xfce-perchannel-xm seed /usr/share/applications/chromium.desktop ~/.config/xfce4/panel/launcher-10/chromium.desktop seed /usr/share/applications/thunar.desktop ~/.config/xfce4/panel/launcher-11/thunar.desktop seed /usr/share/applications/debian-xterm.desktop ~/.config/xfce4/panel/launcher-12/debian-xterm.desktop +# stamp only on a clean seed: a partial force-copy must retry next start, not +# flip to missing-only and leave the desk on a mixed old/new look +[ "$seeded" = 1 ] && mkdir -p "$(dirname "$STAMP")" && echo "$LOOK" > "$STAMP" rm -f /tmp/.X0-lock /tmp/.X11-unix/X0 # stale after docker stop; blocks Xvfb on wake # -fbdir /dev/shm: mmap the framebuffer to a file deskd reads for screenshots @@ -48,6 +68,12 @@ xfdesktop & mkfifo /tmp/.chrome-stdin 2>/dev/null exec 9<>/tmp/.chrome-stdin +# --restore-last-session and no start URL: the volume is the computer's identity, so +# a wake should hand the agent back the tabs it was working in, not a blank browser. +# Passing about:blank here instead would stack one more dead tab on every wake. +# --renderer-process-limit caps process sprawl as tabs accumulate over a desk's life; +# it does mean cross-site tabs can share a renderer, which matters less here than it +# would elsewhere because --no-sandbox has already given up that isolation. chrome_loop() { while true; do chromium <&9 \ @@ -64,7 +90,9 @@ chrome_loop() { --disable-component-extensions-with-background-pages \ --disable-features=NetworkTimeServiceQuerying,OptimizationHints \ --start-maximized \ - about:blank >>/tmp/chromium.log 2>&1 + --restore-last-session \ + --renderer-process-limit=8 \ + >>/tmp/chromium.log 2>&1 echo "[chrome_loop] chromium exited rc=$?" >>/tmp/chromium.log sleep 1 done diff --git a/mcp/case_mcp.py b/mcp/case_mcp.py index 6fea741..242a6d3 100644 --- a/mcp/case_mcp.py +++ b/mcp/case_mcp.py @@ -145,10 +145,12 @@ def computer_eval(computer_id: str, expression: str, timeout_s: int = 20) -> dic def computer_navigate(computer_id: str, url: str, timeout_s: int = 30) -> dict: """Point the computer's browser at url and block until the page has loaded. One call — do not follow it with readyState polling. Returns - {ok, url, title} (url is the final one, after redirects) or {ok:false, error}. + {ok, url, title, snapshot} (url is the final one, after redirects) or + {ok:false, error}. `snapshot` is the arrived page's numbered elements, already + there — DO NOT call computer_snapshot after this, you have it. The body is ready when this returns; `title` is best-effort and can be "" on pages that set it a beat late — that is not a signal to wait or retry. - Then read the page with computer_eval("document.body.innerText"). + Read page text with computer_eval("document.body.innerText"). Same-page '#anchor' jumps are not navigations; use computer_eval for those.""" return call("POST", f"/computers/{computer_id}/navigate", params={"wake": "true"}, json={"url": url, "timeout_s": timeout_s}, @@ -163,8 +165,15 @@ def computer_snapshot(computer_id: str) -> dict: coordinate guessing. Returns {ok, url, title, count, elements} where each element is a line like '[12] button "Save changes"' or '[13] input(email) "" =\\'\\' — pass that number to computer_click_element or - computer_fill. Refs are re-derived per call (document order), so a ref is valid - until the page changes; after a click or navigation, snapshot again. + computer_fill. Everything currently on screen is included; on a big page the rest + is cut to a budget, so `count` can exceed the lines you get and the numbers SKIP — + that is normal, use the number on the line, never its position in the list. Scroll + and snapshot again to reach what was cut. + Refs are re-derived per call (document order), so a ref is valid + until the page changes. You rarely need this tool twice: computer_navigate, + computer_click_element and computer_fill(submit) all return the fresh snapshot + in their own result. Call this for the FIRST look at a page, or after something + changed it that Case did not do (a timer, a redirect you waited out). Screenshots are still right for canvas/custom-drawn UI, visual layout questions, and anything outside the browser window.""" return call("GET", f"/computers/{computer_id}/page", params={"wake": "true"}).json() @@ -180,7 +189,9 @@ def computer_click_element(computer_id: str, ref: int, name: str = None, The element is scrolled into view and clicked with a real OS-level mouse event (isTrusted true). text, when given, is typed into the element after the click (click focuses it) — for one field that beats computer_fill. - One call replaces the screenshot→guess-coordinates→click→screenshot loop.""" + On success the result carries `snapshot`: the page as it stands after the click, + settled. DO NOT call computer_snapshot after this — read the refs from there and + click again. One call is the whole act-then-look loop.""" body = {"ref": ref} if name is not None: body["name"] = name @@ -202,7 +213,8 @@ def computer_fill(computer_id: str, fields: list, submit: bool = False) -> dict: NEVER for passwords or OTP codes — password fields are refused inside the page; vaulted computer_login owns credentials. Per-field results come back in {fields:[{ref, ok, ...}]}; a failed ref usually means the page changed — - re-snapshot.""" + re-snapshot. With submit=true the result also carries `snapshot` of the page the + submit landed on, so don't call computer_snapshot after one.""" body = {"fields": fields} if submit: body["submit"] = True diff --git a/tests/test_browse.py b/tests/test_browse.py index 5dcf300..d34bc8c 100644 --- a/tests/test_browse.py +++ b/tests/test_browse.py @@ -56,6 +56,19 @@ def test_snapshot_formats_numbered_lines(): assert "=" in out["elements"][2], out["elements"] # input value shown +def test_snapshot_numbers_lines_by_document_index_not_list_position(): + """The shown list is the on-screen elements first, so on a big page it is a subset + with gaps. The number on the line is the ref click_element re-derives — take it + from the element, never from where it landed in the list.""" + els = [dict(ELS[0], i=0), dict(ELS[1], i=97), dict(ELS[2], i=412)] + browse.eval_js = fake_eval({"ok": True, "value": { + "url": "u", "title": "t", "count": 3752, "els": els}}) + out = browse.snapshot(ROW) + assert out["truncated"] is True and out["count"] == 3752, out + assert out["elements"][1].startswith("[97] "), out["elements"] + assert out["elements"][2].startswith("[412] "), out["elements"] + + def test_snapshot_truncation_flagged(): browse.eval_js = fake_eval({"ok": True, "value": { "url": "u", "title": "t", "count": 400, "els": ELS}}) @@ -139,10 +152,12 @@ def test_fill_requires_ref_and_value(): def test_fill_passes_fields_and_returns_page_result(): browse.eval_js = fake_eval({"ok": True, "value": { "ok": True, "fields": [{"ref": 2, "ok": True, "name": "Work email"}]}}) - out = browse.fill(ROW, [{"ref": 2, "value": "jane@x.com"}], submit=True) + out = browse.fill(ROW, [{"ref": 2, "value": "jane@x.com"}], submit=True, + snapshot_after=False) # this is about the script, not the settle assert out["ok"] is True, out expr = browse.eval_js.calls[0] assert '"jane@x.com"' in expr and "__submit=true" in expr + assert "out.every(o=>o.ok)" in expr and "out.some" not in expr assert "vault-only" in expr # the password refusal ships inside the page script @@ -270,6 +285,135 @@ def test_teach_tick_502_nav_churn_is_quiet(): assert out["ok"] and out["gap"] == 502, out +# ---------- act then observe: the page rides back with the action ---------- + +def fake_eval_map(rules): + """Answer by what an expression asks for, not by call order — the settle loop + polls an indeterminate number of times. Per-needle results are consumed in order + and the last one repeats.""" + calls, used = [], {k: 0 for k in rules} + + def _eval(row, expression, timeout_s=20): + calls.append(expression) + for needle, results in rules.items(): + if needle in expression: + r = results[min(used[needle], len(results) - 1)] + used[needle] += 1 + if isinstance(r, Exception): + raise r + return r if isinstance(r, dict) else {"ok": True, "value": r} + raise AssertionError(f"unexpected eval: {expression[:90]}") + _eval.calls = calls + return _eval + + +LOCATE = "const __i=" +STAMP = "__case_act=1" +POLL = "__case_act===undefined" +UNSTAMP = "delete window.__case_act" +SNAP = "count:__els.length" +FILL = "const __fields=" + + +def _snap_value(url, els=()): + return {"ok": True, "value": {"url": url, "title": "T", "count": len(els), "els": list(els)}} + + +def test_click_returns_the_page_it_navigated_to(): + # the saving is a whole LLM turn: the agent never has to ask "what happened?" + browse.eval_js = fake_eval_map({ + LOCATE: [{"ok": True, "value": {"ok": True, "name": "Go", "tag": "a", "x": 1, "y": 2}}], + STAMP: [{"ok": True, "value": 1}], + POLL: [{"ok": True, "value": [True, "complete"]}], # stamp gone = new document + SNAP: [_snap_value("https://next.test", ELS[:1])], + }) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Go") + assert out["ok"], out + assert out["snapshot"]["url"] == "https://next.test", out + assert out["snapshot"]["elements"][0] == '[0] a "Home" -> /', out + + +def test_click_snapshot_never_hands_back_the_page_it_left(): + """The bug this stamp exists for: the old document reports readyState 'complete' + for a beat after the click, so polling readyState alone snapshots the page the + agent just navigated away from.""" + browse.eval_js = fake_eval_map({ + LOCATE: [{"ok": True, "value": {"ok": True, "name": "Go", "tag": "a", "x": 1, "y": 2}}], + STAMP: [{"ok": True, "value": 1}], + POLL: [{"ok": True, "value": [False, "complete"]}, # old doc, already complete + {"ok": True, "value": [False, "complete"]}, + ApiError(502, "eval_error", "context destroyed"), # navigation commits + {"ok": True, "value": [True, "loading"]}, + {"ok": True, "value": [True, "complete"]}], + SNAP: [_snap_value("https://next.test")], + }) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Go") + assert out["snapshot"]["url"] == "https://next.test", out + + +def test_click_that_changes_nothing_still_returns_the_current_page(): + # an in-page click never navigates; after the grace, this page IS the answer + browse.eval_js = fake_eval_map({ + LOCATE: [{"ok": True, "value": {"ok": True, "name": "Tab", "tag": "button", + "x": 1, "y": 2}}], + STAMP: [{"ok": True, "value": 1}], + POLL: [{"ok": True, "value": [False, "complete"]}], + SNAP: [_snap_value("https://same.test", ELS)], + UNSTAMP: [{"ok": True, "value": None}], + }) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Tab") + assert out["snapshot"]["url"] == "https://same.test", out + # the page is left as it was found: no uniquely-named global for site JS to see + assert any(UNSTAMP in c for c in browse.eval_js.calls), browse.eval_js.calls + + +def test_click_snapshot_can_be_turned_off(): + browse.eval_js = fake_eval({"ok": True, "value": {"ok": True, "name": "n", "tag": "a", + "x": 1, "y": 2}}) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, snapshot_after=False) + assert out["ok"] and "snapshot" not in out, out + assert len(browse.eval_js.calls) == 1, browse.eval_js.calls # the locate walk, nothing more + + +def test_fill_snapshots_only_when_it_submitted(): + filled = {"ok": True, "value": {"ok": True, "fields": [{"ref": 2, "ok": True}]}} + # a plain fill leaves the caller's refs valid — don't spend the settle on it + browse.eval_js = fake_eval_map({FILL: [filled]}) + out = browse.fill(ROW, [{"ref": 2, "value": "a@b.c"}]) + assert out["ok"] and "snapshot" not in out, out + assert len(browse.eval_js.calls) == 1, browse.eval_js.calls + # a submit moves the page, and that is exactly when the caller is blind + browse.eval_js = fake_eval_map({ + FILL: [filled], + STAMP: [{"ok": True, "value": 1}], + POLL: [{"ok": True, "value": [True, "complete"]}], + SNAP: [_snap_value("https://after.test")], + }) + out = browse.fill(ROW, [{"ref": 2, "value": "a@b.c"}], submit=True) + assert out["snapshot"]["url"] == "https://after.test", out + + +def test_click_unstamped_does_not_treat_missing_marker_as_navigation(): + """If the stamp never landed, __case_act===undefined is this document, not a + new one. Polling it as 'navigated' would snapshot as soon as readyState is + complete, which the page we started on already is.""" + browse.eval_js = fake_eval_map({ + LOCATE: [{"ok": True, "value": {"ok": True, "name": "Tab", "tag": "button", + "x": 1, "y": 2}}], + STAMP: [{"ok": False, "error": "eval failed"}], + "document.readyState": [{"ok": True, "value": "complete"}], + SNAP: [_snap_value("https://same.test", ELS)], + }) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Tab") + assert out["snapshot"]["url"] == "https://same.test", out + assert not any(POLL in c for c in browse.eval_js.calls), browse.eval_js.calls + + def test_teach_tick_504_still_raises(): browse.eval_js = fake_eval(ApiError(504, "daemon_timeout", "deskd did not respond")) try: diff --git a/web/web-ui/case-tools.mjs b/web/web-ui/case-tools.mjs index 8587ac7..cf07680 100644 --- a/web/web-ui/case-tools.mjs +++ b/web/web-ui/case-tools.mjs @@ -12,7 +12,7 @@ import https from 'node:https'; import Anthropic from '@anthropic-ai/sdk'; export const CASE_TOOLS = [ - { type: 'function', name: 'computer_navigate', description: 'Point the computer browser at url and block until the page has loaded. Returns {ok, url, title}. Then read with computer_eval("document.body.innerText"). Same-page #anchor jumps are not navigations.', parameters: { type: 'object', properties: { url: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['url'], additionalProperties: false } }, + { type: 'function', name: 'computer_navigate', description: 'Point the computer browser at url and block until the page has loaded. Returns {ok, url, title, snapshot} — `snapshot` holds the numbered elements of the page it arrived on, so do NOT call computer_snapshot after this. Then read with computer_eval("document.body.innerText"). Same-page #anchor jumps are not navigations.', parameters: { type: 'object', properties: { url: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['url'], additionalProperties: false } }, { type: 'function', name: 'computer_eval', description: 'Evaluate JS in the active tab (CDP, promises awaited). Prefer this over screenshots for page content. Return plain values — DOM nodes are not serialisable. To read a page as prose, document.body.innerText. Do not drive location.assign from here; use computer_navigate.', parameters: { type: 'object', properties: { expression: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['expression'], additionalProperties: false } }, { type: 'function', name: 'computer_action', description: 'UI action on the desktop (1280x800 by default): click|double_click|move|drag|scroll|type|key|wait. Coordinates are pixels, origin top-left. keys uses xdotool syntax (ctrl+l, Return). For elements INSIDE a web page prefer computer_snapshot + computer_click_element; use this for the desktop itself, canvas, shortcuts, and scrolling.', parameters: { type: 'object', properties: { type: { type: 'string', enum: ['click', 'double_click', 'move', 'drag', 'scroll', 'type', 'key', 'wait'] }, x: { type: 'number' }, y: { type: 'number' }, text: { type: 'string' }, keys: { type: 'string' }, dy: { type: 'number' }, ms: { type: 'number' }, from_x: { type: 'number' }, from_y: { type: 'number' }, to_x: { type: 'number' }, to_y: { type: 'number' } }, required: ['type'], additionalProperties: false } }, { type: 'function', name: 'computer_exec', description: 'Run a shell command on the computer (bash, as user agent).', parameters: { type: 'object', properties: { command: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['command'], additionalProperties: false } }, diff --git a/web/web-ui/serve.mjs b/web/web-ui/serve.mjs index a7323db..ee783b6 100644 --- a/web/web-ui/serve.mjs +++ b/web/web-ui/serve.mjs @@ -369,9 +369,9 @@ const EXTRA_TOOLS = [ { type: 'function', name: 'computer_list', description: 'List all computers with state, resources and credential names. Reuse an existing computer — only computer_create for an identity that should stay separate.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, { type: 'function', name: 'computer_create', description: 'Create a persistent computer (Linux desktop + Chromium). Blocks until running. Computers are durable: logins, cookies and files survive sleep. Check computer_list first.', parameters: { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } }, { type: 'function', name: 'computer_screenshot', description: 'Screenshot of the computer display (1280x800 by default). Wakes if asleep. Prefer computer_snapshot for anything inside a web page; use this for canvas, visual layout, and anything outside the browser window.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, - { type: 'function', name: 'computer_snapshot', description: 'Numbered list of visible interactive elements on the active browser tab. PREFER over screenshots for finding what to click: returns lines like [12] button "Save changes" — pass the number to computer_click_element or computer_fill. Re-snapshot after any click or navigation.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, - { type: 'function', name: 'computer_click_element', description: 'Click element [ref] from the last computer_snapshot. Pass name (quoted text from the snapshot line) so a changed page is refused with a fresh snapshot instead of a wrong click. text, if given, is typed into the element after the click.', parameters: { type: 'object', properties: { ref: { type: 'number' }, name: { type: 'string' }, text: { type: 'string' }, screenshot: { type: 'boolean' } }, required: ['ref'], additionalProperties: false } }, - { type: 'function', name: 'computer_fill', description: 'Fill a whole form in one call: fields=[{ref, value}] with refs from computer_snapshot. Never for passwords or OTP codes — vaulted computer_login owns those. submit=true submits the form at the end.', parameters: { type: 'object', properties: { fields: { type: 'array', items: { type: 'object', properties: { ref: { type: 'number' }, value: {} }, required: ['ref', 'value'], additionalProperties: false } }, submit: { type: 'boolean' } }, required: ['fields'], additionalProperties: false } }, + { type: 'function', name: 'computer_snapshot', description: 'Numbered list of visible interactive elements on the active browser tab. PREFER over screenshots for finding what to click: returns lines like [12] button "Save changes" — pass the number to computer_click_element or computer_fill. Everything on screen is included; on a big page the rest is cut to a budget, so the numbers SKIP — use the number on the line, never its position. You rarely need this twice: navigate, click and fill(submit) each return the fresh snapshot themselves.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, + { type: 'function', name: 'computer_click_element', description: 'Click element [ref] from the last computer_snapshot. Pass name (quoted text from the snapshot line) so a changed page is refused with a fresh snapshot instead of a wrong click. text, if given, is typed into the element after the click. The result carries `snapshot`: the settled page the click produced — do NOT call computer_snapshot after this.', parameters: { type: 'object', properties: { ref: { type: 'number' }, name: { type: 'string' }, text: { type: 'string' }, screenshot: { type: 'boolean' } }, required: ['ref'], additionalProperties: false } }, + { type: 'function', name: 'computer_fill', description: 'Fill a whole form in one call: fields=[{ref, value}] with refs from computer_snapshot. Never for passwords or OTP codes — vaulted computer_login owns those. submit=true submits the form at the end, and the result then carries `snapshot` of the page it landed on — no computer_snapshot needed after one.', parameters: { type: 'object', properties: { fields: { type: 'array', items: { type: 'object', properties: { ref: { type: 'number' }, value: {} }, required: ['ref', 'value'], additionalProperties: false } }, submit: { type: 'boolean' } }, required: ['fields'], additionalProperties: false } }, { type: 'function', name: 'computer_wait_for', description: 'Block until the page is ready instead of polling with eval. Exactly one of: selector (CSS), text (in body innerText), network_idle=true. gone=true inverts selector/text (wait for spinner to disappear).', parameters: { type: 'object', properties: { selector: { type: 'string' }, text: { type: 'string' }, gone: { type: 'boolean' }, network_idle: { type: 'boolean' }, timeout_s: { type: 'number' } }, additionalProperties: false } }, { type: 'function', name: 'computer_tabs', description: 'Browser tabs: action=list|activate|new|close. eval/snapshot/capture talk to the ACTIVE tab — if a click opened a new tab and the page stopped responding, list then activate the right one. activate/close need target_id from list; new needs an http(s) url.', parameters: { type: 'object', properties: { action: { type: 'string', enum: ['list', 'activate', 'new', 'close'] }, target_id: { type: 'string' }, url: { type: 'string' } }, required: ['action'], additionalProperties: false } }, { type: 'function', name: 'computer_capture_start', description: 'Start capturing network response bodies in the active tab whose URL matches url_pattern (regex). Survives SPA nav; catches fetch and XHR. Then navigate/act and drain with computer_capture_read. e.g. url_pattern="SearchTimeline|/graphql".', parameters: { type: 'object', properties: { url_pattern: { type: 'string' } }, required: ['url_pattern'], additionalProperties: false } }, @@ -679,7 +679,7 @@ async function chat(req, res) { thread.agent = id; emit({ type: 'claim', thread_id: thread.id, agent: id }); }; - const dev = { role: 'developer', content: `You operate Case computer ${id}${cname ? ` (named "${cname}" — that's you when the user addresses it)` : ''} via tools. Loop: computer_navigate, then computer_snapshot to see numbered clickable elements, then computer_click_element/computer_fill by ref. One snapshot per navigation or state change is enough — refs stay valid until the page changes, so several clicks can follow one snapshot. computer_wait_for instead of polling. Read page text with computer_eval document.body.innerText. Screenshots only for visual layout. Coordinates are the display size (1280x800 by default). Login walls: computer_login(credential=, url=current page) — never ask the user for a password or type into password fields. Vault names on this computer: ${vault}. On handoff_pending, immediately auth_attempt_wait. You get ${ROUNDS} tool steps per turn; the conversation continues across turns, so if you run out say exactly where you stopped. Short final answer.` }; + const dev = { role: 'developer', content: `You operate Case computer ${id}${cname ? ` (named "${cname}" — that's you when the user addresses it)` : ''} via tools. Loop: computer_navigate, then computer_click_element/computer_fill by ref. navigate, click and fill(submit) each RETURN the page's numbered elements, so read the refs from the result you already have — call computer_snapshot only for a first look or when something you did not do changed the page. Refs stay valid until the page changes and the numbers can skip; use the number on the line. computer_wait_for instead of polling. Read page text with computer_eval document.body.innerText. Screenshots only for visual layout. Coordinates are the display size (1280x800 by default). Login walls: computer_login(credential=, url=current page) — never ask the user for a password or type into password fields. Vault names on this computer: ${vault}. On handoff_pending, immediately auth_attempt_wait. You get ${ROUNDS} tool steps per turn; the conversation continues across turns, so if you run out say exactly where you stopped. Short final answer.` }; const hist = thread; const turnStart = hist.items.length; hist.items.push({ role: 'user', content: inputText });