perf: fuse observation into action, and pick snapshot elements by what is on screen

control-plane/browse.py

@@ -11,6 +11,8 @@
   click_element   re-derive the same numbered list, verify the ref still matches,
                   scroll it into view, then fire a real OS-level click through
                   /action at the computed screen coordinates (isTrusted stays true)
+                  — and return the settled page with it, so the caller never has to
+                  spend a second LLM turn asking what the click did
   fill            batch-fill a form via native value setters + input/change events
                   (React-safe); refuses password fields — vault login owns those
   wait_for        server-side 0.4s poll for selector/text/network-idle, so agents
@@ -22,6 +24,13 @@
 emits elements in document order, so a ref from the last snapshot re-derives to
 the same element unless the page itself changed — in which case click_element
 refuses and returns a fresh snapshot instead of clicking the wrong thing.
+
+A ref is that document-order index, which is why snapshot can show any subset it
+likes without breaking anything. It shows the on-screen ones first: a big page has
+thousands of interactive elements and the cap has to fall somewhere, and falling on
+"whoever appears first in the HTML" spends it on chrome the reader scrolled past.
+Measured on a scrolled Wikipedia article: a third of what was on screen missed the
+cut. The shown numbers therefore skip.
 """
 import json
 import re
@@ -34,8 +43,9 @@
 
 MAX_ELS = 150
 
-# The shared walk. Defines __els = [{el, tag, type, name, value, href}] in
-# document order, visible interactive elements only. Password values never read.
+# The shared walk. Defines __els = [{el, tag, type, name, value, href, on}] in
+# document order, visible interactive elements only (`on` = 0 when the element is in
+# the viewport right now). Password values never read.
 _WALK = """
 const __sel='a[href],button,input,select,textarea,summary,label,'+
  '[role="button"],[role="link"],[role="tab"],[role="menuitem"],[role="checkbox"],'+
@@ -55,7 +65,8 @@
     el.innerText||el.title||el.alt||'').trim().replace(/\\s+/g,' ').slice(0,80);
   __els.push({el,tag,type:(el.type||el.getAttribute('role')||''),name,
     value:('value'in el&&el.type!=='password'&&tag!=='button')?String(el.value).slice(0,40):'',
-    href:tag==='a'?String(el.getAttribute('href')||'').slice(0,120):''});
+    href:tag==='a'?String(el.getAttribute('href')||'').slice(0,120):'',
+    on:(r.bottom>0&&r.top<innerHeight&&r.right>0&&r.left<innerWidth)?0:1});
 }
 """
 
@@ -76,9 +87,15 @@ def _fmt(i, e):
 
 
 def snapshot(row, timeout_s=15):
-    """Numbered visible interactive elements of the active tab, document order."""
-    body = ("return {url:location.href,title:document.title,count:__els.length,"
-            "els:__els.slice(0,%d).map(({el,...r})=>r)};" % MAX_ELS)
+    """The active tab's interactive elements as numbered lines, everything on screen
+    first. Refs are indices into the whole walk, so the shown list can skip numbers."""
+    body = ("""
+const __by=__els.map((e,i)=>i);
+__by.sort((a,b)=>__els[a].on-__els[b].on||a-b);   // on screen first, document order within
+const __pick=__by.slice(0,%d).sort((a,b)=>a-b);   // then hand them back in reading order
+return {url:location.href,title:document.title,count:__els.length,
+  els:__pick.map(i=>{const{el,on,...r}=__els[i];return{i,...r};})};
+""" % MAX_ELS)
     r = eval_js(row, _iife(body), timeout_s)
     if not r.get("ok"):
         return {"ok": False, "error": r.get("error") or "snapshot failed"}
@@ -89,7 +106,7 @@ def snapshot(row, timeout_s=15):
     return {"ok": True, "url": v.get("url"), "title": v.get("title"),
             "count": v.get("count", len(els)),
             "truncated": (v.get("count", 0) > MAX_ELS),
-            "elements": [_fmt(i, e) for i, e in enumerate(els)]}
+            "elements": [_fmt(e.get("i", i), e) for i, e in enumerate(els)]}
 
 
 def _locate(row, ref, name, timeout_s=15):
@@ -113,10 +130,96 @@ def _locate(row, ref, name, timeout_s=15):
     return r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad locate result"}
 
 
-def click_element(row, ref, name=None, text=None, screenshot=False):
-    """Verify + scroll + real OS click. On stale ref: refuse and hand back a fresh
-    snapshot (a wrong click is worse than a slow click). text, when given, is typed
-    after the click (the click focuses the field)."""
+# Same sentinel trick deskclient.navigate uses, and for the same reason: a click that
+# navigates leaves the OLD document reporting readyState 'complete' for a beat, so
+# polling readyState alone snapshots the page the agent just left. Stamping the
+# document before the action turns that race into a check — a document without the
+# stamp is, by construction, the one the action produced.
+_STAMP = "window.__case_act"
+
+
+def _stamp(row):
+    try:
+        return bool(eval_js(row, f"{_STAMP}=1", 3).get("ok"))
+    except ApiError:
+        return False
+
+
+def _settled_snapshot(row, stamped, settle_s=1.0, budget_s=8.0):
+    """The page as it stands once the action stops moving it.
+
+    Two endings, and they need telling apart. If the stamp is gone the action
+    navigated: wait for the new document to finish and snapshot that. If the stamp is
+    still there after `settle_s`, nothing navigated and this page IS the answer — a
+    grace that long because a JS-driven navigation does not commit instantly. A 502
+    means the context was torn down mid-navigation, which is progress, so poll through.
+
+    If the stamp never landed, `__case_act===undefined` is the starting document,
+    not proof of navigation. Wait out the grace and snapshot whatever is complete.
+
+    This costs ~2 extra eval round trips at ~11ms each. The computer_snapshot call it
+    saves the agent is a whole model turn. Cheap side of a very lopsided trade.
+    """
+    if not stamped:
+        time.sleep(settle_s)
+        deadline = time.time() + budget_s
+        while time.time() < deadline:
+            time.sleep(0.25)
+            try:
+                r = eval_js(row, "document.readyState", 3)
+            except ApiError as e:
+                if e.status != 502:
+                    return None
+                continue
+            if r.get("value") == "complete":
+                break
+        try:
+            return snapshot(row)
+        except ApiError:
+            return None
+    deadline = time.time() + budget_s
+    grace = time.time() + settle_s
+    while time.time() < deadline:
+        time.sleep(0.25)
+        try:
+            r = eval_js(row, f"[{_STAMP}===undefined,document.readyState]", 3)
+        except ApiError as e:
+            if e.status != 502:
+                return None
+            continue           # context gone: the navigation we are waiting for
+        v = r.get("value")
+        if not isinstance(v, list):
+            break
+        navigated, ready = v[0], v[1]
+        if navigated and ready == "complete":
+            return snapshot(row)
+        if not navigated and time.time() >= grace:
+            fresh = snapshot(row)
+            # don't leave a uniquely-named global on a live page for site JS to find
+            try:
+                eval_js(row, f"delete {_STAMP}", 3)
+            except ApiError:
+                pass
+            return fresh
+    try:
+        return snapshot(row)   # never settled; a partial list still beats a blind turn
+    except ApiError:
+        return None
+
+
+def _attach_snapshot(row, res, want, stamped):
+    if not want:
+        return res
+    fresh = _settled_snapshot(row, stamped)
+    if fresh and fresh.get("ok"):
+        res["snapshot"] = fresh
+    return res
+
+
+def click_element(row, ref, name=None, text=None, screenshot=False, snapshot_after=True):
+    """Verify + scroll + real OS click, then hand back the page it produced. On stale
+    ref: refuse and hand back a fresh snapshot (a wrong click is worse than a slow
+    click). text, when given, is typed after the click (the click focuses the field)."""
     loc = _locate(row, ref, name)
     if not loc.get("ok"):
         if loc.get("stale"):
@@ -129,6 +232,7 @@ def click_element(row, ref, name=None, text=None, screenshot=False):
     x, y = loc["x"], loc["y"]
     if not (0 <= x < SCREEN_W and 0 <= y < SCREEN_H):
         return {"ok": False, "error": f"element resolves off-screen ({x},{y})"}
+    stamped = snapshot_after and _stamp(row)
     out = desk_json(row, "POST", "/action",
                     json={"type": "click", "x": x, "y": y,
                           "screenshot": bool(screenshot) and not text},
@@ -141,10 +245,10 @@ def click_element(row, ref, name=None, text=None, screenshot=False):
     res = {"ok": True, "clicked": loc.get("name"), "tag": loc.get("tag"), "x": x, "y": y}
     if isinstance(out, dict) and out.get("screenshot_png_b64"):
         res["screenshot_png_b64"] = out["screenshot_png_b64"]
-    return res
+    return _attach_snapshot(row, res, snapshot_after, stamped)
 
 
-def fill(row, fields, submit=False, timeout_s=20):
+def fill(row, fields, submit=False, timeout_s=20, snapshot_after=True):
     """Batch form fill. fields=[{ref, value}]. Native setters + input/change events
     so React/Vue see the change. Password inputs are refused inside the page —
     vaulted computer_login owns credentials, always."""
@@ -182,18 +286,23 @@ def fill(row, fields, submit=False, timeout_s=20):
     out.push({ref:f.ref,ok:true,name:e.name});
   }catch(err){out.push({ref:f.ref,ok:false,error:String(err).slice(0,80)});}
 }
-if(__submit&&out.some(o=>o.ok)){
+if(__submit&&out.length&&out.every(o=>o.ok)){
   const first=__els[__fields[0].ref];
   const form=first&&first.el.form;
   if(form){form.requestSubmit?form.requestSubmit():form.submit();}
   else out.push({ref:__fields[0].ref,ok:false,error:'submit requested but field has no form'});
 }
 return {ok:out.every(o=>o.ok),fields:out};
 """
+    # Only the submit moves the page out from under the caller; after a plain fill the
+    # refs they already hold are still good, so don't spend the settle on one.
+    want = bool(snapshot_after and submit)
+    stamped = want and _stamp(row)
     r = eval_js(row, _iife(body), timeout_s)
     if not r.get("ok"):
         return {"ok": False, "error": r.get("error") or "fill failed"}
-    return r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad fill result"}
+    out = r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad fill result"}
+    return _attach_snapshot(row, out, want and out.get("ok"), stamped)
 
 
 def wait_for(row, selector=None, text=None, gone=False, network_idle=False, timeout_s=30):

control-plane/cased.py

@@ -354,7 +354,12 @@ def navigate_(cid: str, body: dict = Body(...), wake: bool = False):
         if "url" not in body:
             raise ApiError(400, "bad_request", "body needs 'url'")
         timeout = max(1, min(int(body.get("timeout_s") or 30), 120))   # never navigate then
-        return navigate(row, body["url"], timeout)                     # report failure at t=0
+        out = navigate(row, body["url"], timeout)                      # report failure at t=0
+        if out.get("ok") and body.get("snapshot", True):
+            fresh = browse.snapshot(row)      # navigate already waited for readyState
+            if fresh.get("ok"):
+                out["snapshot"] = fresh
+        return out
 
 
 # ---------- element-level browsing (browse.py; control-plane composition) ----------
@@ -372,13 +377,15 @@ def click_(cid: str, body: dict = Body(...), wake: bool = False):
             raise ApiError(400, "bad_request", "body needs 'ref' (from GET /page)")
         return browse.click_element(row, int(body["ref"]), name=body.get("name"),
                                     text=body.get("text"),
-                                    screenshot=bool(body.get("screenshot")))
+                                    screenshot=bool(body.get("screenshot")),
+                                    snapshot_after=bool(body.get("snapshot", True)))
 
 
 @app.post("/v1/computers/{cid}/fill")
 def fill_(cid: str, body: dict = Body(...), wake: bool = False):
     with awake(cid, wake) as row:
-        return browse.fill(row, body.get("fields"), submit=bool(body.get("submit")))
+        return browse.fill(row, body.get("fields"), submit=bool(body.get("submit")),
+                           snapshot_after=bool(body.get("snapshot", True)))
 
 
 @app.post("/v1/computers/{cid}/wait")

control-plane/lifecycle.py

@@ -9,6 +9,7 @@
 transitioning it, so it deliberately bypasses the transition guard.
 """
 import secrets
+import time
 
 from config import DESK_H, DESK_W, IMAGE, MAX_RAM_MB, MAX_RUNNING, log
 from errors import ApiError
@@ -210,6 +211,7 @@ def do_wake(cid):
     row = get_computer(cid)
     if row["state"] == "running" and dockerd.container_up(cid):
         return  # DB can lie after a daemon restart, only skip if the container really is up
+    t0 = time.monotonic()
     # Asleep computers don't count against the budget; waking one must, same as create.
     # (create already checks; wake used to bypass the cap and OOM a small box.)
     if row["state"] == "asleep":
@@ -225,7 +227,12 @@ def do_wake(cid):
                                      row["desk_token"])
         desk_port, vnc_port = dockerd.container_ports(dockerd.get_container(cid))
         store.set_ports(cid, desk_port, vnc_port)
+        t_container = time.monotonic() - t0
         deskclient.wait_desk(cid, desk_port, row["desk_token"], 30)
+        # The two halves of a wake bill very differently — container start is docker,
+        # the rest is Chromium coming up. Split them or you tune the wrong one.
+        log.info("wake %s: container %.2fs, deskd healthy %.2fs",
+                 cid, t_container, time.monotonic() - t0)
     except Exception:
         _try_set(cid, "asleep")   # tolerate a concurrent delete here too
         raise

image/Dockerfile

@@ -27,9 +27,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # at startup. Invalid dummy keys keep those services off without the "API keys
 # missing" infobar. Web logins (gmail.com etc.) are cookie logins, unaffected.
 # Policies disable the service machinery itself; matching kill flags in start.sh.
+# MemorySaverModeSavings (1 = balanced) rides along here because it is a policy too:
+# desks are long-lived and now restore their tabs across wakes, so background tabs
+# accumulate. Discarded tabs reload from cookies on return, which agents tolerate.
 RUN printf 'export GOOGLE_API_KEY="no"\nexport GOOGLE_DEFAULT_CLIENT_ID="no"\nexport GOOGLE_DEFAULT_CLIENT_SECRET="no"\n' > /etc/chromium.d/apikeys \
     && mkdir -p /etc/chromium/policies/managed \
-    && printf '%s' '{"BrowserSignin":0,"SyncDisabled":true,"MetricsReportingEnabled":false,"SafeBrowsingProtectionLevel":0,"BackgroundModeEnabled":false,"ComponentUpdatesEnabled":false,"PromotionalTabsEnabled":false,"UrlKeyedAnonymizedDataCollectionEnabled":false,"DefaultSearchProviderEnabled":true,"DefaultSearchProviderName":"DuckDuckGo","DefaultSearchProviderKeyword":"duckduckgo.com","DefaultSearchProviderSearchURL":"https://duckduckgo.com/?q={searchTerms}","DefaultSearchProviderSuggestURL":"https://duckduckgo.com/ac/?q={searchTerms}&type=list"}' \
+    && printf '%s' '{"BrowserSignin":0,"SyncDisabled":true,"MetricsReportingEnabled":false,"SafeBrowsingProtectionLevel":0,"BackgroundModeEnabled":false,"ComponentUpdatesEnabled":false,"PromotionalTabsEnabled":false,"UrlKeyedAnonymizedDataCollectionEnabled":false,"MemorySaverModeSavings":1,"DefaultSearchProviderEnabled":true,"DefaultSearchProviderName":"DuckDuckGo","DefaultSearchProviderKeyword":"duckduckgo.com","DefaultSearchProviderSearchURL":"https://duckduckgo.com/?q={searchTerms}","DefaultSearchProviderSuggestURL":"https://duckduckgo.com/ac/?q={searchTerms}&type=list"}' \
        > /etc/chromium/policies/managed/case.json
 
 ARG NOVNC_VERSION=1.5.0

image/start.sh

@@ -4,14 +4,31 @@
 # Traps SIGTERM (docker stop = sleep) so Chromium exits cleanly and flushes its profile.
 
 RES="${DESK_RESOLUTION:-1280x800x24}"
+# Xvfb needs WxHxD and dies on bare WxH; depth 24 is the only one deskd's
+# XWD->PNG grab supports (32bpp), so it is also the only default we append.
+case "$RES" in *x*x*) ;; *) RES="${RES}x24" ;; esac
 
 # libXcursor honors this in every X client — the one switch that themes the
 # cursor everywhere (chromium reads it via GTK settings too, seeded below).
 export XCURSOR_THEME=case XCURSOR_SIZE=32
 
-# Seed per-user desktop config when missing (first boot, or volumes from older
-# images); never overwrite — the volume is the user's.
-seed() { [ -f "$2" ] || { mkdir -p "$(dirname "$2")"; cp "$1" "$2"; }; }
+# Seed per-user desktop config from /usr/share/case. Two rules fight here: never
+# stomp the user's own tweaks, but a shipped look change has to actually land.
+# LOOK settles it. The volume outlives the image (it IS the computer's identity),
+# so a desk created before a look change keeps writing back its old xfconf files
+# and a missing-only seed silently skips every one of them — leaving the new
+# wallpaper/panel/dock sitting unused in the image. Bump LOOK whenever the
+# assets change: every desk force-seeds exactly once (resetting desktop tweaks
+# that one time), then goes back to leaving the user alone.
+LOOK=v2
+STAMP=~/.config/.case-look
+seeded=1
+if [ "$(cat "$STAMP" 2>/dev/null)" = "$LOOK" ]; then
+  seed() { [ -f "$2" ] || { mkdir -p "$(dirname "$2")" && cp "$1" "$2"; } || seeded=0; }
+else
+  echo "[start] look $LOOK — re-seeding desktop config" >&2
+  seed() { mkdir -p "$(dirname "$2")" && cp -f "$1" "$2" || seeded=0; }
+fi
 seed /usr/share/case/gtk-settings.ini ~/.config/gtk-3.0/settings.ini
 seed /usr/share/case/xfce4-panel.xml ~/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml
 seed /usr/share/case/xfwm4.xml ~/.config/xfce4/xfconf/xfce-perchannel-xml/xfwm4.xml
@@ -20,6 +37,9 @@ seed /usr/share/case/xfce4-desktop.xml ~/.config/xfce4/xfconf/xfce-perchannel-xm
 seed /usr/share/applications/chromium.desktop ~/.config/xfce4/panel/launcher-10/chromium.desktop
 seed /usr/share/applications/thunar.desktop ~/.config/xfce4/panel/launcher-11/thunar.desktop
 seed /usr/share/applications/debian-xterm.desktop ~/.config/xfce4/panel/launcher-12/debian-xterm.desktop
+# stamp only on a clean seed: a partial force-copy must retry next start, not
+# flip to missing-only and leave the desk on a mixed old/new look
+[ "$seeded" = 1 ] && mkdir -p "$(dirname "$STAMP")" && echo "$LOOK" > "$STAMP"
 
 rm -f /tmp/.X0-lock /tmp/.X11-unix/X0   # stale after docker stop; blocks Xvfb on wake
 # -fbdir /dev/shm: mmap the framebuffer to a file deskd reads for screenshots
@@ -48,6 +68,12 @@ xfdesktop &
 mkfifo /tmp/.chrome-stdin 2>/dev/null
 exec 9<>/tmp/.chrome-stdin
 
+# --restore-last-session and no start URL: the volume is the computer's identity, so
+# a wake should hand the agent back the tabs it was working in, not a blank browser.
+# Passing about:blank here instead would stack one more dead tab on every wake.
+# --renderer-process-limit caps process sprawl as tabs accumulate over a desk's life;
+# it does mean cross-site tabs can share a renderer, which matters less here than it
+# would elsewhere because --no-sandbox has already given up that isolation.
 chrome_loop() {
   while true; do
     chromium <&9 \
@@ -64,7 +90,9 @@ chrome_loop() {
       --disable-component-extensions-with-background-pages \
       --disable-features=NetworkTimeServiceQuerying,OptimizationHints \
       --start-maximized \
-      about:blank >>/tmp/chromium.log 2>&1
+      --restore-last-session \
+      --renderer-process-limit=8 \
+      >>/tmp/chromium.log 2>&1
     echo "[chrome_loop] chromium exited rc=$?" >>/tmp/chromium.log
     sleep 1
   done