diff --git a/NOTICE b/NOTICE index 82aeb02..b1ebc78 100644 --- a/NOTICE +++ b/NOTICE @@ -47,6 +47,7 @@ cryptography Apache-2.0 or BSD-3-Clause requests Apache-2.0 websocket-client Apache-2.0 mcp (Model Context Protocol SDK) MIT +Pillow MIT-CMU pytest MIT -------------------------------------------------------------------------------- diff --git a/README.md b/README.md index b954d07..ea4a215 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ brain (Claude, Cursor, Codex, or the Drive UI with your provider key). What the agent gets, over MCP: - **A real desktop**: navigate, snapshot numbered clickable elements, click/fill - by ref, exec, files, network capture — no coordinate guessing. + by ref, hover menus, upload files under `/home/agent`, marked screenshots, + exec, files, network capture — no coordinate guessing. Navigate and click + return the first 2000 characters of page text. - **Vault logins**: the human saves a credential once (encrypted, via a one-time link); the machine types it into the site's own login page. The agent and the API never see the password. diff --git a/SECURITY.md b/SECURITY.md index 150aa2c..fc1f634 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,6 +9,10 @@ Case holds logins. These are promises, with code you can read. - **Login only fires** when the page host matches the credential's `domains`. - **MCP has no credential-write tool.** Secrets enter via the Drive UI, `/fill`, or `bin/case cred add`. +- **`computer_upload` only assigns files already on the computer.** The path + must be under `/home/agent/`, at most 5MB, and the snapshot ref must be + `input[type=file]`. Password/OTP-like inputs are refused. Bytes travel + through deskd `GET /file`, never command stdout. - **cased binds loopback by default.** Compose publishes `127.0.0.1:8787` and `127.0.0.1:4174`. Set `CASE_TOKEN` before exposing those ports. - **Audit log** (`~/.case/audit/.jsonl`): one line per API call; request diff --git a/control-plane/browse.py b/control-plane/browse.py index 93ff3dd..ebb012c 100644 --- a/control-plane/browse.py +++ b/control-plane/browse.py @@ -11,8 +11,6 @@ click_element re-derive the same numbered list, verify the ref still matches, scroll it into view, then fire a real OS-level click through /action at the computed screen coordinates (isTrusted stays true) - — and return the settled page with it, so the caller never has to - spend a second LLM turn asking what the click did fill batch-fill a form via native value setters + input/change events (React-safe); refuses password fields — vault login owns those wait_for server-side 0.4s poll for selector/text/network-idle, so agents @@ -24,50 +22,99 @@ emits elements in document order, so a ref from the last snapshot re-derives to the same element unless the page itself changed — in which case click_element refuses and returns a fresh snapshot instead of clicking the wrong thing. - -A ref is that document-order index, which is why snapshot can show any subset it -likes without breaking anything. It shows the on-screen ones first: a big page has -thousands of interactive elements and the cap has to fall somewhere, and falling on -"whoever appears first in the HTML" spends it on chrome the reader scrolled past. -Measured on a scrolled Wikipedia article: a third of what was on screen missed the -cut. The shown numbers therefore skip. """ import json +import posixpath import re import shlex import time +from urllib.parse import quote from config import DESK_W as SCREEN_W, DESK_H as SCREEN_H from errors import ApiError -from deskclient import desk_json, eval_js +import base64 + +from deskclient import desk_bytes, desk_json, eval_js, page_text MAX_ELS = 150 -# The shared walk. Defines __els = [{el, tag, type, name, value, href, on}] in -# document order, visible interactive elements only (`on` = 0 when the element is in -# the viewport right now). Password values never read. +# The shared walk. Defines __els = [{el, tag, type, name, value, href, where, …}] +# in document order. Pierces open shadow roots and same-origin iframes; includes +# cursor:pointer / onclick widgets; drops occluded nodes and children whose bbox +# sits inside an already-emitted hit target. Password/OTP values are never read. _WALK = """ +const __secretish=el=>el.type==='password' + ||['one-time-code','current-password','new-password'].includes((el.getAttribute&&el.getAttribute('autocomplete'))||''); const __sel='a[href],button,input,select,textarea,summary,label,'+ '[role="button"],[role="link"],[role="tab"],[role="menuitem"],[role="checkbox"],'+ '[role="radio"],[role="combobox"],[role="option"],[role="switch"],[role="searchbox"],'+ '[role="textbox"],[onclick],[tabindex],[contenteditable="true"]'; -const __seen=new Set(),__els=[]; -for(const el of document.querySelectorAll(__sel)){ - if(__seen.has(el))continue;__seen.add(el); - if(el.closest('[aria-hidden="true"]'))continue; - if(el.disabled)continue; +const __seen=new Set(),__els=[],__boxes=[]; +const __clickable=el=>{ + try{if(el.matches&&el.matches(__sel))return true;}catch(e){} + if(el.onclick)return true; + try{if(getComputedStyle(el).cursor==='pointer')return true;}catch(e){} + return false; +}; +const __vis=el=>{ + if(el.disabled)return false; + try{if(el.closest&&el.closest('[aria-hidden="true"]'))return false;}catch(e){} + const r=el.getBoundingClientRect(); + if(r.width<2||r.height<2)return false; + if(typeof el.checkVisibility==='function'&&!el.checkVisibility())return false; + return true; +}; +const __occluded=el=>{ const r=el.getBoundingClientRect(); - if(r.width<2||r.height<2)continue; - if(typeof el.checkVisibility==='function'&&!el.checkVisibility())continue; + const x=r.left+r.width/2,y=r.top+r.height/2; + const root=el.getRootNode(); + let hit=null; + try{hit=root.elementFromPoint?root.elementFromPoint(x,y):document.elementFromPoint(x,y);}catch(e){return false;} + if(!hit)return false; + return hit!==el&&!el.contains(hit)&&!hit.contains(el); +}; +const __contained=r=>__boxes.some(b=>r.x>=b.x&&r.x+r.w<=b.x+b.w&&r.y>=b.y&&r.y+r.h<=b.y+b.h); +const __topRect=el=>{ + let r=el.getBoundingClientRect(),x=r.left,y=r.top; + let w=el.ownerDocument&&el.ownerDocument.defaultView; + while(w&&w.frameElement){ + const fr=w.frameElement.getBoundingClientRect(); + x+=fr.left;y+=fr.top; + w=w.frameElement.ownerDocument&&w.frameElement.ownerDocument.defaultView; + } + return {x,y,w:r.width,h:r.height}; +}; +const __push=(el,where)=>{ + if(__seen.has(el)||!__clickable(el)||!__vis(el)||__occluded(el))return; + const r=el.getBoundingClientRect(); + const tr=__topRect(el); + if(__contained(tr))return; + __seen.add(el); + __boxes.push(tr); const tag=el.tagName.toLowerCase(); const name=(el.getAttribute('aria-label')||el.placeholder|| ((tag==='input'&&(el.type==='submit'||el.type==='button'))?el.value:'')|| el.innerText||el.title||el.alt||'').trim().replace(/\\s+/g,' ').slice(0,80); + const bx=(window.outerWidth-window.innerWidth)/2; __els.push({el,tag,type:(el.type||el.getAttribute('role')||''),name, - value:('value'in el&&el.type!=='password'&&tag!=='button')?String(el.value).slice(0,40):'', + value:('value'in el&&!__secretish(el)&&tag!=='button')?String(el.value).slice(0,40):'', href:tag==='a'?String(el.getAttribute('href')||'').slice(0,120):'', - on:(r.bottom>0&&r.top0&&r.left{ + let nodes=[]; + try{nodes=root.querySelectorAll('*');}catch(e){return;} + for(const el of nodes){ + __push(el,where); + if(el.shadowRoot)__walk(el.shadowRoot,'shadow'); + if((el.tagName||'')==='IFRAME'){ + try{if(el.contentDocument)__walk(el.contentDocument,'iframe');}catch(e){} + } + } +}; +__walk(document,''); """ @@ -75,10 +122,12 @@ def _iife(body): return "(()=>{" + _WALK + body + "})()" -def _fmt(i, e): +def _fmt(i, e, star=False): """One element, one compact line: [12] button 'Save changes'.""" t = e["tag"] + ("(" + e["type"] + ")" if e["type"] and e["type"] != e["tag"] else "") - line = f'[{i}] {t} "{e["name"]}"' + mark = f"|{e['where']}| " if e.get("where") else "" + pref = "*" if star else "" + line = f'{pref}[{i}] {mark}{t} "{e["name"]}"' if e.get("value"): line += f' ={e["value"]!r}' if e.get("href"): @@ -87,15 +136,14 @@ def _fmt(i, e): def snapshot(row, timeout_s=15): - """The active tab's interactive elements as numbered lines, everything on screen - first. Refs are indices into the whole walk, so the shown list can skip numbers.""" - body = (""" -const __by=__els.map((e,i)=>i); -__by.sort((a,b)=>__els[a].on-__els[b].on||a-b); // on screen first, document order within -const __pick=__by.slice(0,%d).sort((a,b)=>a-b); // then hand them back in reading order + """Numbered visible interactive elements of the active tab, document order.""" + body = """ +const keys=__els.map(e=>e.tag+'\\0'+e.name+'\\0'+(e.href||'')); +const prev=(window.__caseEls&&window.__caseEls.keys)||[]; +window.__caseEls={els:__els.map(e=>e.el),keys:keys}; return {url:location.href,title:document.title,count:__els.length, - els:__pick.map(i=>{const{el,on,...r}=__els[i];return{i,...r};})}; -""" % MAX_ELS) + els:__els.slice(0,%d).map(({el,...r})=>r),keys:keys.slice(0,%d),prev:prev.slice(0,%d)}; +""" % (MAX_ELS, MAX_ELS, MAX_ELS) r = eval_js(row, _iife(body), timeout_s) if not r.get("ok"): return {"ok": False, "error": r.get("error") or "snapshot failed"} @@ -103,26 +151,49 @@ def snapshot(row, timeout_s=15): if not isinstance(v, dict): # >64KB truncates to a string; should not happen at 150 els return {"ok": False, "error": "snapshot too large — page has an extreme DOM"} els = v.get("els") or [] + keys = v.get("keys") or [] + prev = set(v.get("prev") or []) + starred = bool(prev) return {"ok": True, "url": v.get("url"), "title": v.get("title"), "count": v.get("count", len(els)), "truncated": (v.get("count", 0) > MAX_ELS), - "elements": [_fmt(e.get("i", i), e) for i, e in enumerate(els)]} + "elements": [_fmt(i, e, star=starred and i < len(keys) and keys[i] not in prev) + for i, e in enumerate(els)]} def _locate(row, ref, name, timeout_s=15): """Re-derive the walk, verify ref+name, scroll into view, return screen coords.""" checks = f"const __i={int(ref)};const __n={json.dumps(name) if name else 'null'};" body = checks + """ -if(__i<0||__i>=__els.length)return {ok:false,stale:true,count:__els.length}; -const e=__els[__i]; -if(__n!==null&&e.name!==__n&&!e.name.includes(__n)) - return {ok:false,stale:true,found:e.name,count:__els.length}; -e.el.scrollIntoView({block:'center',inline:'center'}); -const r=e.el.getBoundingClientRect(); -const bx=(window.outerWidth-window.innerWidth)/2; -return {ok:true,name:e.name,tag:e.tag, - x:Math.round(window.screenX+bx+r.left+r.width/2), - y:Math.round(window.screenY+(window.outerHeight-window.innerHeight)-bx+r.top+r.height/2)}; +const __nameOk=(nm)=>__n===null||nm===__n||(nm&&nm.includes(__n)); +const __coords=(el,nm,tag,healed,oldI,newI)=>{ + el.scrollIntoView({block:'center',inline:'center'}); + const tr=__topRect(el); + const bx=(window.outerWidth-window.innerWidth)/2; + return {ok:true,name:nm,tag:tag,type:(el.type||''),healed:!!healed,old_ref:oldI,new_ref:newI, + x:Math.round(window.screenX+bx+tr.x+tr.w/2), + y:Math.round(window.screenY+(window.outerHeight-window.innerHeight)-bx+tr.y+tr.h/2)}; +}; +const stored=window.__caseEls&&window.__caseEls.els&&window.__caseEls.els[__i]; +if(stored&&stored.isConnected){ + const tag=stored.tagName.toLowerCase(); + const nm=(stored.getAttribute('aria-label')||stored.placeholder||stored.innerText||'').trim().replace(/\\s+/g,' ').slice(0,80); + if(__nameOk(nm))return __coords(stored,nm,tag,false,__i,__i); +} +if(__i>=0&&__i<__els.length){ + const e=__els[__i]; + if(__nameOk(e.name))return __coords(e.el,e.name,e.tag,false,__i,__i); + if(__n!==null){ + const hits=__els.map((x,i)=>({x,i})).filter(h=>h.x.name===__n||h.x.name.includes(__n)); + if(hits.length===1)return __coords(hits[0].x.el,hits[0].x.name,hits[0].x.tag,true,__i,hits[0].i); + return {ok:false,stale:true,found:e.name,count:__els.length}; + } +} +if(__n!==null){ + const hits=__els.map((x,i)=>({x,i})).filter(h=>h.x.name===__n||h.x.name.includes(__n)); + if(hits.length===1)return __coords(hits[0].x.el,hits[0].x.name,hits[0].x.tag,true,__i,hits[0].i); +} +return {ok:false,stale:true,count:__els.length}; """ r = eval_js(row, _iife(body), timeout_s) if not r.get("ok"): @@ -130,11 +201,9 @@ def _locate(row, ref, name, timeout_s=15): return r.get("value") if isinstance(r.get("value"), dict) else {"ok": False, "error": "bad locate result"} -# Same sentinel trick deskclient.navigate uses, and for the same reason: a click that -# navigates leaves the OLD document reporting readyState 'complete' for a beat, so -# polling readyState alone snapshots the page the agent just left. Stamping the -# document before the action turns that race into a check — a document without the -# stamp is, by construction, the one the action produced. +# Same sentinel as deskclient.navigate: a click that navigates leaves the old +# document reporting readyState complete for a beat. Stamp first so a document +# without the stamp is the one the action produced. _STAMP = "window.__case_act" @@ -146,20 +215,7 @@ def _stamp(row): def _settled_snapshot(row, stamped, settle_s=1.0, budget_s=8.0): - """The page as it stands once the action stops moving it. - - Two endings, and they need telling apart. If the stamp is gone the action - navigated: wait for the new document to finish and snapshot that. If the stamp is - still there after `settle_s`, nothing navigated and this page IS the answer — a - grace that long because a JS-driven navigation does not commit instantly. A 502 - means the context was torn down mid-navigation, which is progress, so poll through. - - If the stamp never landed, `__case_act===undefined` is the starting document, - not proof of navigation. Wait out the grace and snapshot whatever is complete. - - This costs ~2 extra eval round trips at ~11ms each. The computer_snapshot call it - saves the agent is a whole model turn. Cheap side of a very lopsided trade. - """ + """The page as it stands once the action stops moving it.""" if not stamped: time.sleep(settle_s) deadline = time.time() + budget_s @@ -186,7 +242,7 @@ def _settled_snapshot(row, stamped, settle_s=1.0, budget_s=8.0): except ApiError as e: if e.status != 502: return None - continue # context gone: the navigation we are waiting for + continue v = r.get("value") if not isinstance(v, list): break @@ -195,14 +251,13 @@ def _settled_snapshot(row, stamped, settle_s=1.0, budget_s=8.0): return snapshot(row) if not navigated and time.time() >= grace: fresh = snapshot(row) - # don't leave a uniquely-named global on a live page for site JS to find try: eval_js(row, f"delete {_STAMP}", 3) except ApiError: pass return fresh try: - return snapshot(row) # never settled; a partial list still beats a blind turn + return snapshot(row) except ApiError: return None @@ -232,6 +287,7 @@ def click_element(row, ref, name=None, text=None, screenshot=False, snapshot_aft x, y = loc["x"], loc["y"] if not (0 <= x < SCREEN_W and 0 <= y < SCREEN_H): return {"ok": False, "error": f"element resolves off-screen ({x},{y})"} + before = _page_ids(row) if loc.get("tag") in ("a", "button") and text is None else [] stamped = snapshot_after and _stamp(row) out = desk_json(row, "POST", "/action", json={"type": "click", "x": x, "y": y, @@ -243,15 +299,154 @@ def click_element(row, ref, name=None, text=None, screenshot=False, snapshot_aft "screenshot": bool(screenshot)}, timeout=30) res = {"ok": True, "clicked": loc.get("name"), "tag": loc.get("tag"), "x": x, "y": y} + if loc.get("healed"): + res["healed"] = True + res["old_ref"] = loc.get("old_ref") + res["new_ref"] = loc.get("new_ref") + if before: + switched = _activate_new_tab(row, before) + if switched: + res["switched_tab"] = switched if isinstance(out, dict) and out.get("screenshot_png_b64"): res["screenshot_png_b64"] = out["screenshot_png_b64"] - return _attach_snapshot(row, res, snapshot_after, stamped) + res = _attach_snapshot(row, res, snapshot_after, stamped) + t = page_text(row, eval_js) + if t: + res["text"] = t + return res + + +def _page_ids(row): + try: + return [t["id"] for t in tabs(row).get("tabs") or [] if t.get("id")] + except Exception: + return [] + + +def _activate_new_tab(row, before): + try: + after = tabs(row).get("tabs") or [] + except Exception: + return None + known = set(before) + fresh = [t for t in after if t.get("id") and t["id"] not in known] + if len(fresh) != 1: + return None + t = fresh[0] + try: + tabs(row, action="activate", target_id=t["id"]) + except Exception: + return None + return {"id": t["id"], "url": t.get("url"), "title": t.get("title")} + + +def overlay_marks(png, rects): + """Draw numbered boxes on a desktop PNG. Never injects into the live DOM. + Returns the original bytes if Pillow cannot read the image.""" + try: + from io import BytesIO + from PIL import Image, ImageDraw + im = Image.open(BytesIO(png)).convert("RGB") + dr = ImageDraw.Draw(im) + for i, e in enumerate(rects or []): + x, y, w, h = e.get("sx"), e.get("sy"), e.get("vw"), e.get("vh") + if None in (x, y, w, h): + continue + x, y, w, h = int(x), int(y), int(w), int(h) + dr.rectangle([x, y, x + w, y + h], outline=(220, 40, 40), width=2) + dr.text((x + 2, max(0, y - 12)), str(i), fill=(220, 40, 40)) + buf = BytesIO() + im.save(buf, format="PNG") + return buf.getvalue() + except Exception: + return png + + +def element_rects(row, timeout_s=15): + body = ("return __els.slice(0,%d).map((e,i)=>({i:i,sx:e.sx,sy:e.sy,vw:e.vw,vh:e.vh}));" + % MAX_ELS) + r = eval_js(row, _iife(body), timeout_s) + if not r.get("ok") or not isinstance(r.get("value"), list): + return [] + return r["value"] + + +def hover(row, ref, name=None): + """Move the OS pointer over [ref] without clicking.""" + loc = _locate(row, ref, name) + if not loc.get("ok"): + if loc.get("stale"): + return {"ok": False, "stale": True, "error": "element list changed", + "snapshot": snapshot(row)} + return loc + x, y = loc["x"], loc["y"] + if not (0 <= x < SCREEN_W and 0 <= y < SCREEN_H): + return {"ok": False, "error": f"element resolves off-screen ({x},{y})"} + desk_json(row, "POST", "/action", json={"type": "move", "x": x, "y": y}, timeout=30) + return {"ok": True, "hovered": loc.get("name"), "tag": loc.get("tag"), "x": x, "y": y} + + +UPLOAD_MAX = 5 * 1024 * 1024 +_UPLOAD_CHUNK = 6000 + + +def upload(row, ref, path, name=None): + """Assign a file already on the computer to input[type=file] [ref].""" + p = str(path or "") + if "\n" in p or not p.startswith("/home/agent/") or any(part == ".." for part in p.split("/")): + raise ApiError(400, "bad_request", "path must be under /home/agent/") + p = posixpath.normpath(p) + if not p.startswith("/home/agent/"): + raise ApiError(400, "bad_request", "path must be under /home/agent/") + st = desk_json(row, "POST", "/exec", + json={"command": f"test -f {shlex.quote(p)} && wc -c < {shlex.quote(p)}", + "timeout_s": 10}, timeout=20) + raw = (st.get("stdout") or "").strip().split()[0] if st.get("exit_code") == 0 else "" + try: + size = int(raw) + except ValueError: + raise ApiError(400, "bad_request", "file not found on the computer") + if size > UPLOAD_MAX: + raise ApiError(400, "bad_request", f"file larger than {UPLOAD_MAX} bytes") + loc = _locate(row, ref, name) + if not loc.get("ok"): + if loc.get("stale"): + return {"ok": False, "stale": True, "error": "element list changed", + "snapshot": snapshot(row)} + return loc + use_ref = int(loc["new_ref"]) if loc.get("new_ref") is not None else int(ref) + if loc.get("type") != "file": + return {"ok": False, "error": "ref is not input[type=file] — snapshot again"} + fname = p.rsplit("/", 1)[-1] + raw = desk_bytes(row, "GET", "/file", params={"path": p}, timeout=120) + if len(raw) != size: + return {"ok": False, "error": "file size changed during read"} + data = base64.b64encode(raw).decode("ascii") + eval_js(row, "window.__caseUp=''", 5) + for i in range(0, len(data), _UPLOAD_CHUNK): + eval_js(row, "window.__caseUp+=%s" % json.dumps(data[i:i + _UPLOAD_CHUNK]), 10) + done = eval_js(row, _iife(f""" +const __i={use_ref}; +const stored=window.__caseEls&&window.__caseEls.els&&window.__caseEls.els[__i]; +const e=(__els[__i])||(stored?{{el:stored,type:stored.type}}:null); +if(!e||e.type!=='file')return {{ok:false,error:'not a file input'}}; +const raw=atob(window.__caseUp||'');delete window.__caseUp; +if(raw.length!=={int(size)})return {{ok:false,error:'decoded length mismatch'}}; +const u=new Uint8Array(raw.length); +for(let i=0;io.ok)){ @@ -294,8 +498,6 @@ def fill(row, fields, submit=False, timeout_s=20, snapshot_after=True): } return {ok:out.every(o=>o.ok),fields:out}; """ - # Only the submit moves the page out from under the caller; after a plain fill the - # refs they already hold are still good, so don't spend the settle on one. want = bool(snapshot_after and submit) stamped = want and _stamp(row) r = eval_js(row, _iife(body), timeout_s) @@ -421,7 +623,7 @@ def tabs(row, action="list", target_id=None, url=None): if action == "new": if not url or not re.match(r"^https?://", str(url)): raise ApiError(400, "bad_request", "need an http(s) url") - raw = _cdp_curl(row, "/json/new?" + url, method="PUT") + raw = _cdp_curl(row, "/json/new?" + quote(str(url), safe=""), method="PUT") made = None try: made = json.loads(raw) diff --git a/control-plane/cased.py b/control-plane/cased.py index 48282bb..fedd502 100644 --- a/control-plane/cased.py +++ b/control-plane/cased.py @@ -318,10 +318,16 @@ def awake(cid, wake): @app.get("/v1/computers/{cid}/screenshot") -def screenshot(cid: str, wake: bool = False): +def screenshot(cid: str, wake: bool = False, marks: bool = False): with awake(cid, wake) as row: # desk_bytes raises ApiError(423) during credential injection - return Response(desk_bytes(row, "GET", "/screenshot"), media_type="image/png") + content = desk_bytes(row, "GET", "/screenshot") + if marks: + try: + content = browse.overlay_marks(content, browse.element_rects(row)) + except Exception: + pass + return Response(content, media_type="image/png") @app.post("/v1/computers/{cid}/action") @@ -381,6 +387,22 @@ def click_(cid: str, body: dict = Body(...), wake: bool = False): snapshot_after=bool(body.get("snapshot", True))) +@app.post("/v1/computers/{cid}/hover") +def hover_(cid: str, body: dict = Body(...), wake: bool = False): + with awake(cid, wake) as row: + if "ref" not in body: + raise ApiError(400, "bad_request", "body needs 'ref' (from GET /page)") + return browse.hover(row, int(body["ref"]), name=body.get("name")) + + +@app.post("/v1/computers/{cid}/upload") +def upload_(cid: str, body: dict = Body(...), wake: bool = False): + with awake(cid, wake) as row: + if "ref" not in body or "path" not in body: + raise ApiError(400, "bad_request", "body needs 'ref' and 'path'") + return browse.upload(row, int(body["ref"]), body["path"], name=body.get("name")) + + @app.post("/v1/computers/{cid}/fill") def fill_(cid: str, body: dict = Body(...), wake: bool = False): with awake(cid, wake) as row: diff --git a/control-plane/deskclient.py b/control-plane/deskclient.py index 2db3901..13106c1 100644 --- a/control-plane/deskclient.py +++ b/control-plane/deskclient.py @@ -71,6 +71,65 @@ def eval_value(row, expression, timeout_s=15, default=None): # *existing* container (lifecycle.do_wake), so a new deskd route would reach only # computers created after an image rebuild. This reaches every computer that # exists today, on a cased restart. +_PAGE_TEXT = "document.body?document.body.innerText.slice(0,2000):''" + + +def page_text(row, _eval=None): + """Best-effort first 2000 chars of the page. None on any failure so a + navigation or click that already happened still returns what it did. + `_eval` is the eval_js to use — browse patches its own copy.""" + fn = _eval or eval_js + try: + r = fn(row, _PAGE_TEXT, 3) + t = r.get("value") if r.get("ok") else None + return t if isinstance(t, str) and t else None + except ApiError: + return None + + +def _with_text(row, arrived): + t = page_text(row) + if t: + arrived["text"] = t + return arrived + + +_HYDRATE_N = ("document.readyState==='complete'" + "?document.querySelectorAll('a,button,input,select,textarea').length:0") + + +def _hydrate(row, arrived, deadline): + """SPA: readyState can complete before React mounts any controls.""" + time.sleep(min(2.0, max(0.0, deadline - time.time()))) + try: + r = eval_js(row, _HYDRATE_N, 3) + n = r.get("value") if r.get("ok") else 0 + if n: + return _with_text(row, arrived) + except ApiError as e: + if e.status != 502: + raise + if time.time() >= deadline: + return _with_text(row, arrived) + try: + eval_js(row, "location.reload()", 10) + except ApiError as e: + if e.status != 502: + raise + return _with_text(row, arrived) + while time.time() < deadline: + time.sleep(0.4) + try: + r = eval_js(row, _HYDRATE_N, 3) + except ApiError as e: + if e.status != 502: + raise + continue + if r.get("ok") and r.get("value"): + return _with_text(row, arrived) + return _with_text(row, arrived) + + def navigate(row, url, timeout_s=30): """Load `url` in the browser tab and block until the new document is ready. @@ -91,7 +150,8 @@ def navigate(row, url, timeout_s=30): """ deadline = time.time() + timeout_s poll = ("window.__case_nav===undefined&&document.readyState==='complete'" - "?[location.href,document.title]:null") + "?[location.href,document.title," + "document.querySelectorAll('a,button,input,select,textarea').length]:null") while True: try: r = eval_js(row, f"window.__case_nav=1;location.assign({json.dumps(url)})", 10) @@ -113,8 +173,11 @@ def navigate(row, url, timeout_s=30): raise # asleep / injecting / wedged, not something to wait out continue # context torn down mid-navigation: that IS progress v = r.get("value") - if isinstance(v, list): # a >64KB result comes back as a truncated *string* - return {"ok": True, "url": v[0], "title": v[1]} + if isinstance(v, list) and len(v) >= 2: # a >64KB result is a truncated *string* + arrived = {"ok": True, "url": v[0], "title": v[1]} + if len(v) > 2 and v[2] == 0: + return _hydrate(row, arrived, deadline) + return _with_text(row, arrived) try: # don't leave a uniquely-named global on a live page for site JS to find eval_js(row, "delete window.__case_nav", 3) except ApiError: diff --git a/mcp/case_mcp.py b/mcp/case_mcp.py index 242a6d3..6676eef 100644 --- a/mcp/case_mcp.py +++ b/mcp/case_mcp.py @@ -78,10 +78,13 @@ def computer_list() -> dict: @mcp.tool() -def computer_screenshot(computer_id: str) -> Image: +def computer_screenshot(computer_id: str, marks: bool = False) -> Image: """Screenshot of the computer's display (1280x800 by default; see computer_list - display for the actual size). Wakes the computer if asleep.""" - r = call("GET", f"/computers/{computer_id}/screenshot", params={"wake": "true"}) + display for the actual size). Wakes the computer if asleep. + marks=true draws numbered boxes matching snapshot refs (drawn on the PNG in + the control plane — the live page is not modified).""" + r = call("GET", f"/computers/{computer_id}/screenshot", + params={"wake": "true", "marks": "true" if marks else "false"}) return Image(data=r.content, format="png") @@ -145,12 +148,12 @@ def computer_eval(computer_id: str, expression: str, timeout_s: int = 20) -> dic def computer_navigate(computer_id: str, url: str, timeout_s: int = 30) -> dict: """Point the computer's browser at url and block until the page has loaded. One call — do not follow it with readyState polling. Returns - {ok, url, title, snapshot} (url is the final one, after redirects) or - {ok:false, error}. `snapshot` is the arrived page's numbered elements, already - there — DO NOT call computer_snapshot after this, you have it. + {ok, url, title, text, snapshot} (url is the final one, after redirects) or + {ok:false, error}. `text` is the first 2000 chars of the page; only eval + innerText when you need more. `snapshot` is the arrived page's numbered + elements — DO NOT call computer_snapshot after this, you have it. The body is ready when this returns; `title` is best-effort and can be "" on pages that set it a beat late — that is not a signal to wait or retry. - Read page text with computer_eval("document.body.innerText"). Same-page '#anchor' jumps are not navigations; use computer_eval for those.""" return call("POST", f"/computers/{computer_id}/navigate", params={"wake": "true"}, json={"url": url, "timeout_s": timeout_s}, @@ -165,12 +168,10 @@ def computer_snapshot(computer_id: str) -> dict: coordinate guessing. Returns {ok, url, title, count, elements} where each element is a line like '[12] button "Save changes"' or '[13] input(email) "" =\\'\\' — pass that number to computer_click_element or - computer_fill. Everything currently on screen is included; on a big page the rest - is cut to a budget, so `count` can exceed the lines you get and the numbers SKIP — - that is normal, use the number on the line, never its position in the list. Scroll - and snapshot again to reach what was cut. - Refs are re-derived per call (document order), so a ref is valid - until the page changes. You rarely need this tool twice: computer_navigate, + computer_fill. Starred lines (`*[n]`) appeared since the last snapshot + (autocomplete/typeahead); click one, do not press Enter. Refs are + re-derived per call (document order), so a ref is valid until the page + changes. You rarely need this tool twice: computer_navigate, computer_click_element and computer_fill(submit) all return the fresh snapshot in their own result. Call this for the FIRST look at a page, or after something changed it that Case did not do (a timer, a redirect you waited out). @@ -184,14 +185,16 @@ def computer_click_element(computer_id: str, ref: int, name: str = None, text: str = None, screenshot: bool = False) -> dict: """Click element [ref] from the last computer_snapshot. Pass name (the quoted text from the snapshot line) so a changed page is caught: on mismatch this - REFUSES to click and returns {ok:false, stale:true, snapshot} — use the fresh - snapshot and retry with the right ref; never click blind after a refusal. + REFUSES to click and returns {ok:false, stale:true, snapshot} unless exactly + one current element matches name — then it heals and clicks ({healed:true}). + Use the fresh snapshot after a refusal; never click blind. The element is scrolled into view and clicked with a real OS-level mouse event - (isTrusted true). text, when given, is typed into the element after the click + (isTrusted true). A click that opens a new tab activates it and returns + switched_tab. Returns the first 2000 chars of page text after the click. + text, when given, is typed into the element after the click (click focuses it) — for one field that beats computer_fill. - On success the result carries `snapshot`: the page as it stands after the click, - settled. DO NOT call computer_snapshot after this — read the refs from there and - click again. One call is the whole act-then-look loop.""" + On success the result also carries `snapshot`: the page as it stands after the + click, settled. DO NOT call computer_snapshot after this.""" body = {"ref": ref} if name is not None: body["name"] = name @@ -203,6 +206,29 @@ def computer_click_element(computer_id: str, ref: int, name: str = None, params={"wake": "true"}, json=body, timeout=60).json() +@mcp.tool() +def computer_hover(computer_id: str, ref: int, name: str = None) -> dict: + """Hover the OS pointer over snapshot [ref] without clicking — opens menus + that only appear on hover. Pass name so a changed page is refused.""" + body = {"ref": ref} + if name is not None: + body["name"] = name + return call("POST", f"/computers/{computer_id}/hover", + params={"wake": "true"}, json=body, timeout=40).json() + + +@mcp.tool() +def computer_upload(computer_id: str, ref: int, path: str, name: str = None) -> dict: + """Assign a file already on the computer (path under /home/agent/, ≤5MB) to + snapshot [ref], which must be input[type=file]. Never send file bytes through + this tool — write the file with computer_file_put or computer_exec first.""" + body = {"ref": ref, "path": path} + if name is not None: + body["name"] = name + return call("POST", f"/computers/{computer_id}/upload", + params={"wake": "true"}, json=body, timeout=90).json() + + @mcp.tool() def computer_fill(computer_id: str, fields: list, submit: bool = False) -> dict: """Fill a whole form in ONE call: fields=[{"ref": 13, "value": "jane@x.com"}, …] diff --git a/requirements.txt b/requirements.txt index 0c03eab..1bddc0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ docker==7.1.0 cryptography==49.0.0 requests==2.34.2 mcp==1.28.1 +Pillow==12.2.0 diff --git a/tests/test_browse.py b/tests/test_browse.py index d34bc8c..67e572f 100644 --- a/tests/test_browse.py +++ b/tests/test_browse.py @@ -4,6 +4,7 @@ Run: .venv/bin/python tests/test_browse.py""" import os import sys +from urllib.parse import quote sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "control-plane")) os.environ.setdefault("CASE_HOME", "/tmp/case-browse-test") @@ -56,19 +57,6 @@ def test_snapshot_formats_numbered_lines(): assert "=" in out["elements"][2], out["elements"] # input value shown -def test_snapshot_numbers_lines_by_document_index_not_list_position(): - """The shown list is the on-screen elements first, so on a big page it is a subset - with gaps. The number on the line is the ref click_element re-derives — take it - from the element, never from where it landed in the list.""" - els = [dict(ELS[0], i=0), dict(ELS[1], i=97), dict(ELS[2], i=412)] - browse.eval_js = fake_eval({"ok": True, "value": { - "url": "u", "title": "t", "count": 3752, "els": els}}) - out = browse.snapshot(ROW) - assert out["truncated"] is True and out["count"] == 3752, out - assert out["elements"][1].startswith("[97] "), out["elements"] - assert out["elements"][2].startswith("[412] "), out["elements"] - - def test_snapshot_truncation_flagged(): browse.eval_js = fake_eval({"ok": True, "value": { "url": "u", "title": "t", "count": 400, "els": ELS}}) @@ -86,7 +74,38 @@ def test_snapshot_walk_never_reads_password_values(): browse.eval_js = fake_eval({"ok": True, "value": {"url": "u", "title": "t", "count": 0, "els": []}}) browse.snapshot(ROW) - assert "el.type!=='password'" in browse.eval_js.calls[0] + src = browse.eval_js.calls[0] + assert "__secretish" in src + assert "one-time-code" in src + assert "current-password" in src + assert "!__secretish(el)" in src + + +def test_walk_v2_pierces_shadow_iframe_and_filters(): + browse.eval_js = fake_eval({"ok": True, "value": {"url": "u", "title": "t", + "count": 0, "els": []}}) + browse.snapshot(ROW) + src = browse.eval_js.calls[0] + assert "el.shadowRoot" in src + assert "contentDocument" in src + assert "elementFromPoint" in src + assert "cursor==='pointer'" in src + assert "__contained" in src + assert "__contained(tr)" in src + assert "__boxes.push(tr)" in src + assert "__contained(r)" not in src + + +def test_snapshot_formats_shadow_and_iframe_markers(): + browse.eval_js = fake_eval({"ok": True, "value": { + "url": "u", "title": "t", "count": 2, "els": [ + {"tag": "button", "type": "button", "name": "In shadow", "value": "", + "href": "", "where": "shadow"}, + {"tag": "a", "type": "", "name": "In frame", "value": "", + "href": "/x", "where": "iframe"}]}}) + out = browse.snapshot(ROW) + assert out["elements"][0] == '[0] |shadow| button "In shadow"', out["elements"] + assert out["elements"][1] == '[1] |iframe| a "In frame" -> /x', out["elements"] # ---------- click_element ---------- @@ -95,10 +114,10 @@ def test_click_fires_os_click_at_located_coords(): browse.eval_js = fake_eval({"ok": True, "value": { "ok": True, "name": "Save changes", "tag": "button", "x": 640, "y": 402}}) browse.desk_json = fake_desk({"ok": True}) - out = browse.click_element(ROW, 1, name="Save changes") + out = browse.click_element(ROW, 1, name="Save changes", snapshot_after=False) assert out["ok"] and out["clicked"] == "Save changes", out - m, p, j = browse.desk_json.calls[0] - assert p == "/action" and j["type"] == "click" and (j["x"], j["y"]) == (640, 402), j + clicks = [j for _, p, j in browse.desk_json.calls if p == "/action"] + assert clicks and clicks[0]["type"] == "click" and (clicks[0]["x"], clicks[0]["y"]) == (640, 402), clicks def test_stale_ref_refuses_and_returns_fresh_snapshot(): @@ -124,7 +143,7 @@ def test_click_with_text_types_after_click(): browse.eval_js = fake_eval({"ok": True, "value": { "ok": True, "name": "Work email", "tag": "input", "x": 100, "y": 100}}) browse.desk_json = fake_desk({"ok": True}, {"ok": True}) - browse.click_element(ROW, 2, text="jane@x.com") + browse.click_element(ROW, 2, text="jane@x.com", snapshot_after=False) kinds = [j["type"] for _, _, j in browse.desk_json.calls] assert kinds == ["click", "type"], kinds assert browse.desk_json.calls[1][2]["text"] == "jane@x.com" @@ -134,7 +153,7 @@ def test_name_is_json_quoted_into_the_expression(): browse.eval_js = fake_eval({"ok": True, "value": {"ok": True, "name": "x", "tag": "a", "x": 1, "y": 1}}) browse.desk_json = fake_desk({"ok": True}) - browse.click_element(ROW, 0, name='a"b\'c') + browse.click_element(ROW, 0, name='a"b\'c', snapshot_after=False) assert '"a\\"b\'c"' in browse.eval_js.calls[0] @@ -157,8 +176,21 @@ def test_fill_passes_fields_and_returns_page_result(): assert out["ok"] is True, out expr = browse.eval_js.calls[0] assert '"jane@x.com"' in expr and "__submit=true" in expr - assert "out.every(o=>o.ok)" in expr and "out.some" not in expr - assert "vault-only" in expr # the password refusal ships inside the page script + assert "out.every(o=>o.ok)" in expr + assert "out.some(o=>o.ok)" not in expr + assert "__secretish" in expr and "one-time-code" in expr + assert "vault-only" in expr # the secret-field refusal ships inside the page script + assert "page reformatted value" in expr + assert "actual" in expr + + +def test_fill_mismatch_is_returned_to_the_caller(): + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": False, "fields": [{"ref": 2, "ok": False, "actual": "JANE", + "error": "page reformatted value"}]}}) + out = browse.fill(ROW, [{"ref": 2, "value": "jane"}], snapshot_after=False) + assert out["ok"] is False, out + assert out["fields"][0]["actual"] == "JANE" # ---------- wait_for ---------- @@ -302,6 +334,8 @@ def _eval(row, expression, timeout_s=20): if isinstance(r, Exception): raise r return r if isinstance(r, dict) else {"ok": True, "value": r} + if "innerText.slice" in expression: + return {"ok": True, "value": ""} raise AssertionError(f"unexpected eval: {expression[:90]}") _eval.calls = calls return _eval @@ -334,6 +368,23 @@ def test_click_returns_the_page_it_navigated_to(): assert out["snapshot"]["elements"][0] == '[0] a "Home" -> /', out +def test_click_reads_text_after_the_destination_settles(): + browse.eval_js = fake_eval_map({ + LOCATE: [{"ok": True, "value": {"ok": True, "name": "Go", "tag": "a", "x": 1, "y": 2}}], + STAMP: [{"ok": True, "value": 1}], + POLL: [{"ok": True, "value": [True, "complete"]}], + SNAP: [_snap_value("https://next.test", ELS[:1])], + "innerText.slice": [{"ok": True, "value": "destination copy"}], + }) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Go") + assert out["snapshot"]["url"] == "https://next.test", out + assert out["text"] == "destination copy", out + snap_i = next(i for i, expr in enumerate(browse.eval_js.calls) if SNAP in expr) + text_i = next(i for i, expr in enumerate(browse.eval_js.calls) if "innerText.slice" in expr) + assert snap_i < text_i, browse.eval_js.calls + + def test_click_snapshot_never_hands_back_the_page_it_left(): """The bug this stamp exists for: the old document reports readyState 'complete' for a beat after the click, so polling readyState alone snapshots the page the @@ -376,7 +427,8 @@ def test_click_snapshot_can_be_turned_off(): browse.desk_json = fake_desk({"ok": True}) out = browse.click_element(ROW, 0, snapshot_after=False) assert out["ok"] and "snapshot" not in out, out - assert len(browse.eval_js.calls) == 1, browse.eval_js.calls # the locate walk, nothing more + # locate + optional page-text eval; no settle stamp + assert all("__case_act=1" not in c for c in browse.eval_js.calls), browse.eval_js.calls def test_fill_snapshots_only_when_it_submitted(): @@ -414,6 +466,200 @@ def test_click_unstamped_does_not_treat_missing_marker_as_navigation(): assert not any(POLL in c for c in browse.eval_js.calls), browse.eval_js.calls +def test_snapshot_stars_only_new_keys_after_a_prior_snapshot(): + browse.eval_js = fake_eval({"ok": True, "value": { + "url": "u", "title": "t", "count": 2, + "els": [ + {"tag": "input", "type": "text", "name": "Search", "value": "", "href": ""}, + {"tag": "option", "type": "", "name": "Alice", "value": "", "href": ""}], + "keys": ["input\0Search\0", "option\0Alice\0"], + "prev": ["input\0Search\0"]}}) + out = browse.snapshot(ROW) + assert out["elements"][0].startswith("[0]"), out["elements"] + assert out["elements"][1].startswith("*[1]"), out["elements"] + + +def test_heal_clicks_when_exactly_one_name_matches(): + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": True, "name": "Save changes", "tag": "button", "x": 10, "y": 10, + "healed": True, "old_ref": 1, "new_ref": 4}}) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 1, name="Save changes", snapshot_after=False) + assert out["ok"] and out["healed"] is True, out + assert (out["old_ref"], out["new_ref"]) == (1, 4) + + +def test_locate_script_prefers_stored_handle_then_heal(): + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": True, "name": "x", "tag": "a", "x": 1, "y": 1}}) + browse.desk_json = fake_desk({"ok": True}) + browse.click_element(ROW, 0, name="x", snapshot_after=False) + src = browse.eval_js.calls[0] + assert "window.__caseEls" in src + assert "isConnected" in src + assert "healed" in src + + +def test_click_activates_a_new_tab(): + listing0 = '[{"type":"page","id":"AA11","title":"One","url":"https://a"}]' + listing1 = ('[{"type":"page","id":"BB22","title":"Two","url":"https://b"},' + '{"type":"page","id":"AA11","title":"One","url":"https://a"}]') + listing2 = listing1 + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": True, "name": "Docs", "tag": "a", "x": 10, "y": 10}}) + browse.desk_json = fake_desk( + {"stdout": listing0}, + {"ok": True}, + {"stdout": listing1}, + {"stdout": ""}, + {"stdout": listing2}, + ) + out = browse.click_element(ROW, 0, name="Docs", snapshot_after=False) + assert out["ok"] and out.get("switched_tab", {}).get("id") == "BB22", out + cmds = [c[2].get("command", "") for c in browse.desk_json.calls if c[1] == "/exec"] + assert any("/json/activate/BB22" in x for x in cmds), cmds + + +def test_click_returns_page_text(): + browse.eval_js = fake_eval( + {"ok": True, "value": {"ok": True, "name": "Save changes", "tag": "button", "x": 10, "y": 10}}, + {"ok": True, "value": "after click"}, + ) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 1, name="Save changes", snapshot_after=False) + assert out["ok"] and out["text"] == "after click", out + + +def test_click_omits_text_when_eval_fails(): + browse.eval_js = fake_eval( + {"ok": True, "value": {"ok": True, "name": "Go", "tag": "a", "x": 10, "y": 10}}, + ApiError(502, "eval_error", "gone"), + ) + browse.desk_json = fake_desk({"ok": True}) + out = browse.click_element(ROW, 0, name="Go", snapshot_after=False) + assert out["ok"] and "text" not in out, out + + +def test_hover_moves_without_click(): + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": True, "name": "Menu", "tag": "div", "x": 40, "y": 50}}) + browse.desk_json = fake_desk({"ok": True}) + out = browse.hover(ROW, 3, name="Menu") + assert out["ok"] and out["hovered"] == "Menu", out + assert browse.desk_json.calls[0][2]["type"] == "move" + assert browse.desk_json.calls[0][2]["x"] == 40 + assert all(j.get("type") != "click" for _, p, j in browse.desk_json.calls if p == "/action") + + +def test_upload_refuses_path_outside_home_agent(): + try: + browse.upload(ROW, 0, "/etc/passwd") + except ApiError as e: + assert e.status == 400 + return + assert False + + +def test_upload_refuses_parent_segment(): + try: + browse.upload(ROW, 0, "/home/agent/foo/../etc/passwd") + except ApiError as e: + assert e.status == 400 + return + assert False + + +def test_upload_allows_dotdot_in_filename(): + browse.desk_json = fake_desk({"exit_code": 1, "stdout": ""}) + try: + browse.upload(ROW, 0, "/home/agent/report..final.pdf") + except ApiError as e: + assert e.status == 400 + assert "file not found" in e.message + return + assert False + + +def test_upload_refuses_oversize(): + browse.desk_json = fake_desk({"exit_code": 0, "stdout": str(browse.UPLOAD_MAX + 1)}) + try: + browse.upload(ROW, 0, "/home/agent/big.bin") + except ApiError as e: + assert e.status == 400 and "larger" in e.message + return + assert False + + +def test_upload_refuses_non_file_input(): + browse.desk_json = fake_desk({"exit_code": 0, "stdout": "12"}) + browse.eval_js = fake_eval( + {"ok": True, "value": {"ok": True, "name": "Go", "tag": "button", + "type": "submit", "x": 1, "y": 1, "new_ref": 1}}) + out = browse.upload(ROW, 1, "/home/agent/a.pdf") + assert out["ok"] is False and "file" in out["error"], out + + +def test_upload_reads_via_file_get_not_exec_stdout(): + browse.desk_json = fake_desk({"exit_code": 0, "stdout": "4"}) + browse.desk_bytes = lambda *a, **k: b"ABCD" + browse.eval_js = fake_eval( + {"ok": True, "value": {"ok": True, "name": "Attach", "tag": "input", + "type": "file", "x": 1, "y": 1, "new_ref": 7}}, + {"ok": True, "value": None}, + {"ok": True, "value": None}, + {"ok": True, "value": {"ok": True, "name": "a.txt", "bytes": 4}}) + out = browse.upload(ROW, 0, "/home/agent/a.txt") + assert out["ok"] is True and out["bytes"] == 4, out + assert any("const __i=7" in e for e in browse.eval_js.calls) + assert any("window.__caseUp+=" in e for e in browse.eval_js.calls) + assert any("DataTransfer" in e for e in browse.eval_js.calls) + assert not any("base64" in (c[2] or {}).get("command", "") + for c in browse.desk_json.calls if c[1] == "/exec") + + +def test_upload_rejects_size_mismatch_from_file_get(): + browse.desk_json = fake_desk({"exit_code": 0, "stdout": "4"}) + browse.desk_bytes = lambda *a, **k: b"AB" + browse.eval_js = fake_eval({"ok": True, "value": { + "ok": True, "name": "Attach", "tag": "input", "type": "file", + "x": 1, "y": 1, "new_ref": 0}}) + out = browse.upload(ROW, 0, "/home/agent/a.txt") + assert out["ok"] is False and "size changed" in out["error"], out + + +def test_tabs_new_encodes_query_in_cdp_path(): + listing = '[{"type":"page","id":"T1","title":"x","url":"https://x.com/search?q=1"}]' + browse.desk_json = fake_desk( + {"stdout": '{"id":"T1","type":"page"}'}, + {"stdout": listing}, + ) + url = "https://x.com/search?q=1&src=y" + out = browse.tabs(ROW, action="new", url=url) + assert out["ok"], out + cmd = browse.desk_json.calls[0][2]["command"] + assert "/json/new?" in cmd + assert "q=1&src=y" not in cmd + assert quote(url, safe="") in cmd + + +def test_overlay_marks_draws_index_on_png(): + from PIL import Image + from io import BytesIO + im = Image.new("RGB", (200, 80), (255, 255, 255)) + buf = BytesIO() + im.save(buf, format="PNG") + out = browse.overlay_marks(buf.getvalue(), [ + {"sx": 10, "sy": 10, "vw": 40, "vh": 20}, + {"sx": 80, "sy": 30, "vw": 30, "vh": 15}]) + marked = Image.open(BytesIO(out)) + assert marked.size == (200, 80) + assert marked.getpixel((10, 10)) != (255, 255, 255) + + +def test_overlay_marks_falls_back_on_bad_png(): + assert browse.overlay_marks(b"not-a-png", [{"sx": 1, "sy": 1, "vw": 2, "vh": 2}]) == b"not-a-png" + + def test_teach_tick_504_still_raises(): browse.eval_js = fake_eval(ApiError(504, "daemon_timeout", "deskd did not respond")) try: diff --git a/tests/test_navigate.py b/tests/test_navigate.py index 93dddc7..7cd903d 100644 --- a/tests/test_navigate.py +++ b/tests/test_navigate.py @@ -123,6 +123,61 @@ def test_url_is_json_quoted_into_the_expression(): assert '"https://x.test/a\'b\\"c"' in ev.seen[0], ev.seen[0] +def test_empty_interactive_count_hydrates_and_reloads(): + ev = fake([ + ["https://x.test/", "T", 0], + 0, + {"ok": True, "value": None}, + 4, + ]) + deskclient.eval_js = ev + out = deskclient.navigate(ROW, "https://x.test", timeout_s=8) + assert out["ok"] is True, out + assert any("location.reload()" in e for e in ev.seen), ev.seen + assert any("querySelectorAll('a,button,input,select,textarea')" in e for e in ev.seen) + + +def test_two_element_arrival_skips_hydrate(): + ev = fake([["https://x.test/final", "Title"]]) + deskclient.eval_js = ev + out = deskclient.navigate(ROW, "https://x.test", timeout_s=5) + assert out["ok"] is True + assert not any("location.reload()" in e for e in ev.seen) + + +def test_arrival_carries_page_text(): + deskclient.eval_js = fake([ + ["https://x.test/final", "Title"], + "Hello from the page", + ]) + out = deskclient.navigate(ROW, "https://x.test", timeout_s=5) + assert out["ok"] is True + assert out["text"] == "Hello from the page", out + + +def test_hydrate_arrival_carries_page_text(): + ev = fake([ + ["https://x.test/", "T", 0], + 4, + "hydrated body", + ]) + deskclient.eval_js = ev + out = deskclient.navigate(ROW, "https://x.test", timeout_s=8) + assert out["ok"] is True + assert out["text"] == "hydrated body", out + assert not any("location.reload()" in e for e in ev.seen) + + +def test_page_text_eval_failure_is_omitted(): + deskclient.eval_js = fake([ + ["https://x.test/final", "Title"], + ApiError(502, "eval_error", "context destroyed"), + ]) + out = deskclient.navigate(ROW, "https://x.test", timeout_s=5) + assert out["ok"] is True + assert "text" not in out, out + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_"): diff --git a/web/web-ui/case-tools.mjs b/web/web-ui/case-tools.mjs index cf07680..9802ea4 100644 --- a/web/web-ui/case-tools.mjs +++ b/web/web-ui/case-tools.mjs @@ -12,7 +12,7 @@ import https from 'node:https'; import Anthropic from '@anthropic-ai/sdk'; export const CASE_TOOLS = [ - { type: 'function', name: 'computer_navigate', description: 'Point the computer browser at url and block until the page has loaded. Returns {ok, url, title, snapshot} — `snapshot` holds the numbered elements of the page it arrived on, so do NOT call computer_snapshot after this. Then read with computer_eval("document.body.innerText"). Same-page #anchor jumps are not navigations.', parameters: { type: 'object', properties: { url: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['url'], additionalProperties: false } }, + { type: 'function', name: 'computer_navigate', description: 'Point the computer browser at url and block until the page has loaded. Returns {ok, url, title, text, snapshot} — text is the first 2000 chars of the page; snapshot holds the numbered elements, so do NOT call computer_snapshot after this. Same-page #anchor jumps are not navigations.', parameters: { type: 'object', properties: { url: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['url'], additionalProperties: false } }, { type: 'function', name: 'computer_eval', description: 'Evaluate JS in the active tab (CDP, promises awaited). Prefer this over screenshots for page content. Return plain values — DOM nodes are not serialisable. To read a page as prose, document.body.innerText. Do not drive location.assign from here; use computer_navigate.', parameters: { type: 'object', properties: { expression: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['expression'], additionalProperties: false } }, { type: 'function', name: 'computer_action', description: 'UI action on the desktop (1280x800 by default): click|double_click|move|drag|scroll|type|key|wait. Coordinates are pixels, origin top-left. keys uses xdotool syntax (ctrl+l, Return). For elements INSIDE a web page prefer computer_snapshot + computer_click_element; use this for the desktop itself, canvas, shortcuts, and scrolling.', parameters: { type: 'object', properties: { type: { type: 'string', enum: ['click', 'double_click', 'move', 'drag', 'scroll', 'type', 'key', 'wait'] }, x: { type: 'number' }, y: { type: 'number' }, text: { type: 'string' }, keys: { type: 'string' }, dy: { type: 'number' }, ms: { type: 'number' }, from_x: { type: 'number' }, from_y: { type: 'number' }, to_x: { type: 'number' }, to_y: { type: 'number' } }, required: ['type'], additionalProperties: false } }, { type: 'function', name: 'computer_exec', description: 'Run a shell command on the computer (bash, as user agent).', parameters: { type: 'object', properties: { command: { type: 'string' }, timeout_s: { type: 'number' } }, required: ['command'], additionalProperties: false } }, diff --git a/web/web-ui/serve.mjs b/web/web-ui/serve.mjs index ee783b6..1649284 100644 --- a/web/web-ui/serve.mjs +++ b/web/web-ui/serve.mjs @@ -368,9 +368,11 @@ async function power(res, req, action) { const EXTRA_TOOLS = [ { type: 'function', name: 'computer_list', description: 'List all computers with state, resources and credential names. Reuse an existing computer — only computer_create for an identity that should stay separate.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, { type: 'function', name: 'computer_create', description: 'Create a persistent computer (Linux desktop + Chromium). Blocks until running. Computers are durable: logins, cookies and files survive sleep. Check computer_list first.', parameters: { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } }, - { type: 'function', name: 'computer_screenshot', description: 'Screenshot of the computer display (1280x800 by default). Wakes if asleep. Prefer computer_snapshot for anything inside a web page; use this for canvas, visual layout, and anything outside the browser window.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, - { type: 'function', name: 'computer_snapshot', description: 'Numbered list of visible interactive elements on the active browser tab. PREFER over screenshots for finding what to click: returns lines like [12] button "Save changes" — pass the number to computer_click_element or computer_fill. Everything on screen is included; on a big page the rest is cut to a budget, so the numbers SKIP — use the number on the line, never its position. You rarely need this twice: navigate, click and fill(submit) each return the fresh snapshot themselves.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, - { type: 'function', name: 'computer_click_element', description: 'Click element [ref] from the last computer_snapshot. Pass name (quoted text from the snapshot line) so a changed page is refused with a fresh snapshot instead of a wrong click. text, if given, is typed into the element after the click. The result carries `snapshot`: the settled page the click produced — do NOT call computer_snapshot after this.', parameters: { type: 'object', properties: { ref: { type: 'number' }, name: { type: 'string' }, text: { type: 'string' }, screenshot: { type: 'boolean' } }, required: ['ref'], additionalProperties: false } }, + { type: 'function', name: 'computer_screenshot', description: 'Screenshot of the computer display (1280x800 by default). Wakes if asleep. Prefer computer_snapshot for anything inside a web page; use this for canvas, visual layout, and anything outside the browser window. marks=true draws numbered snapshot boxes on the PNG (does not change the live page).', parameters: { type: 'object', properties: { marks: { type: 'boolean' } }, additionalProperties: false } }, + { type: 'function', name: 'computer_snapshot', description: 'Numbered list of visible interactive elements on the active browser tab. PREFER over screenshots for finding what to click: returns lines like [12] button "Save changes" — pass the number to computer_click_element or computer_fill. Starred lines (*[n]) appeared since the last snapshot. You rarely need this twice: navigate, click and fill(submit) each return the fresh snapshot themselves.', parameters: { type: 'object', properties: {}, additionalProperties: false } }, + { type: 'function', name: 'computer_click_element', description: 'Click element [ref] from the last computer_snapshot. Pass name (quoted text from the snapshot line) so a changed page is refused with a fresh snapshot instead of a wrong click. text, if given, is typed into the element after the click. The result carries `snapshot` and the first 2000 chars of page text — do NOT call computer_snapshot after this.', parameters: { type: 'object', properties: { ref: { type: 'number' }, name: { type: 'string' }, text: { type: 'string' }, screenshot: { type: 'boolean' } }, required: ['ref'], additionalProperties: false } }, + { type: 'function', name: 'computer_hover', description: 'Hover the OS pointer over snapshot [ref] without clicking. Use for menus that only appear on hover. Pass name so a changed page is refused.', parameters: { type: 'object', properties: { ref: { type: 'number' }, name: { type: 'string' } }, required: ['ref'], additionalProperties: false } }, + { type: 'function', name: 'computer_upload', description: 'Assign a file already on the computer (path under /home/agent/, max 5MB) to snapshot [ref] which must be input[type=file]. Write the file first with computer_file_put or exec — do not send bytes here.', parameters: { type: 'object', properties: { ref: { type: 'number' }, path: { type: 'string' }, name: { type: 'string' } }, required: ['ref', 'path'], additionalProperties: false } }, { type: 'function', name: 'computer_fill', description: 'Fill a whole form in one call: fields=[{ref, value}] with refs from computer_snapshot. Never for passwords or OTP codes — vaulted computer_login owns those. submit=true submits the form at the end, and the result then carries `snapshot` of the page it landed on — no computer_snapshot needed after one.', parameters: { type: 'object', properties: { fields: { type: 'array', items: { type: 'object', properties: { ref: { type: 'number' }, value: {} }, required: ['ref', 'value'], additionalProperties: false } }, submit: { type: 'boolean' } }, required: ['fields'], additionalProperties: false } }, { type: 'function', name: 'computer_wait_for', description: 'Block until the page is ready instead of polling with eval. Exactly one of: selector (CSS), text (in body innerText), network_idle=true. gone=true inverts selector/text (wait for spinner to disappear).', parameters: { type: 'object', properties: { selector: { type: 'string' }, text: { type: 'string' }, gone: { type: 'boolean' }, network_idle: { type: 'boolean' }, timeout_s: { type: 'number' } }, additionalProperties: false } }, { type: 'function', name: 'computer_tabs', description: 'Browser tabs: action=list|activate|new|close. eval/snapshot/capture talk to the ACTIVE tab — if a click opened a new tab and the page stopped responding, list then activate the right one. activate/close need target_id from list; new needs an http(s) url.', parameters: { type: 'object', properties: { action: { type: 'string', enum: ['list', 'activate', 'new', 'close'] }, target_id: { type: 'string' }, url: { type: 'string' } }, required: ['action'], additionalProperties: false } }, @@ -393,9 +395,11 @@ export function extraPlan(name, args, id) { const p = `/computers/${encodeURIComponent(id)}`; if (name === 'computer_list') return { method: 'GET', rel: '/computers', act: 'list computers' }; if (name === 'computer_create') return { method: 'POST', rel: '/computers', body: a.name ? { name: a.name } : {}, timeoutMs: 90000, act: `create ${a.name || 'desk'}` }; - if (name === 'computer_screenshot') return { method: 'GET', rel: `${p}/screenshot?wake=true`, screenshot: true, act: 'screenshot' }; + if (name === 'computer_screenshot') return { method: 'GET', rel: `${p}/screenshot?wake=true${a.marks ? '&marks=true' : ''}`, screenshot: true, act: a.marks ? 'screenshot marks' : 'screenshot' }; if (name === 'computer_snapshot') return { method: 'GET', rel: `${p}/page?wake=true`, act: 'snapshot' }; if (name === 'computer_click_element') return { method: 'POST', rel: `${p}/click?wake=true`, body: a, act: `click [${a.ref}]${a.name ? ' ' + a.name : ''}` }; + if (name === 'computer_hover') return { method: 'POST', rel: `${p}/hover?wake=true`, body: a, act: `hover [${a.ref}]` }; + if (name === 'computer_upload') return { method: 'POST', rel: `${p}/upload?wake=true`, body: a, timeoutMs: 90000, act: `upload ${a.path || ''} → [${a.ref}]` }; if (name === 'computer_fill') return { method: 'POST', rel: `${p}/fill?wake=true`, body: a, act: `fill ${Array.isArray(a.fields) ? a.fields.length : 0} fields` }; if (name === 'computer_wait_for') return { method: 'POST', rel: `${p}/wait?wake=true`, body: a, timeoutMs: ((a.timeout_s || 30) + 20) * 1000, act: `wait ${a.selector || a.text || 'network idle'}` }; if (name === 'computer_tabs') return { method: 'POST', rel: `${p}/tabs?wake=true`, body: a, act: `tabs ${a.action || 'list'}` }; @@ -679,7 +683,7 @@ async function chat(req, res) { thread.agent = id; emit({ type: 'claim', thread_id: thread.id, agent: id }); }; - const dev = { role: 'developer', content: `You operate Case computer ${id}${cname ? ` (named "${cname}" — that's you when the user addresses it)` : ''} via tools. Loop: computer_navigate, then computer_click_element/computer_fill by ref. navigate, click and fill(submit) each RETURN the page's numbered elements, so read the refs from the result you already have — call computer_snapshot only for a first look or when something you did not do changed the page. Refs stay valid until the page changes and the numbers can skip; use the number on the line. computer_wait_for instead of polling. Read page text with computer_eval document.body.innerText. Screenshots only for visual layout. Coordinates are the display size (1280x800 by default). Login walls: computer_login(credential=, url=current page) — never ask the user for a password or type into password fields. Vault names on this computer: ${vault}. On handoff_pending, immediately auth_attempt_wait. You get ${ROUNDS} tool steps per turn; the conversation continues across turns, so if you run out say exactly where you stopped. Short final answer.` }; + const dev = { role: 'developer', content: `You operate Case computer ${id}${cname ? ` (named "${cname}" — that's you when the user addresses it)` : ''} via tools. Loop: computer_navigate, then computer_click_element/computer_fill by ref. computer_hover for menus that only open on hover. computer_upload for input[type=file] (file already under /home/agent — write it first). If the snapshot has *[n] lines, those just appeared (autocomplete) — click one, do not press Enter. navigate, click and fill(submit) each RETURN the page's numbered elements and the first 2000 chars of page text, so read the result you already have — call computer_snapshot only for a first look or when something you did not do changed the page. Refs stay valid until the page changes. computer_wait_for instead of polling. Screenshots only for canvas/layout; marks=true draws numbered snapshot boxes on the PNG. Coordinates are the display size (1280x800 by default). Login walls: computer_login(credential=, url=current page) — never ask the user for a password or type into password fields. Vault names on this computer: ${vault}. On handoff_pending, immediately auth_attempt_wait. You get ${ROUNDS} tool steps per turn; the conversation continues across turns, so if you run out say exactly where you stopped. Short final answer.` }; const hist = thread; const turnStart = hist.items.length; hist.items.push({ role: 'user', content: inputText }); diff --git a/web/web-ui/test_serve.mjs b/web/web-ui/test_serve.mjs index 20cfbaa..db492f2 100644 --- a/web/web-ui/test_serve.mjs +++ b/web/web-ui/test_serve.mjs @@ -150,6 +150,14 @@ const shot = extraPlan('computer_screenshot', {}, 'c_ab'); assert.equal(shot.method, 'GET'); assert.equal(shot.rel, '/computers/c_ab/screenshot?wake=true'); assert.ok(shot.screenshot); +const marked = extraPlan('computer_screenshot', { marks: true }, 'c_ab'); +assert.ok(marked.rel.includes('marks=true')); +const hover = extraPlan('computer_hover', { ref: 3, name: 'Menu' }, 'c_ab'); +assert.equal(hover.method, 'POST'); +assert.equal(hover.rel, '/computers/c_ab/hover?wake=true'); +const up = extraPlan('computer_upload', { ref: 2, path: '/home/agent/a.pdf' }, 'c_ab'); +assert.equal(up.method, 'POST'); +assert.equal(up.rel, '/computers/c_ab/upload?wake=true'); const capStart = extraPlan('computer_capture_start', { url_pattern: 'graphql' }, 'c_ab'); assert.equal(capStart.method, 'POST'); assert.equal(capStart.rel, '/computers/c_ab/capture?wake=true');