Add hover, upload, marked shots, and richer page walk

NOTICE

@@ -47,6 +47,7 @@ cryptography                            Apache-2.0 or BSD-3-Clause
 requests                                Apache-2.0
 websocket-client                        Apache-2.0
 mcp (Model Context Protocol SDK)        MIT
+Pillow                                  MIT-CMU
 pytest                                  MIT
 
 --------------------------------------------------------------------------------

README.md

@@ -9,7 +9,9 @@ brain (Claude, Cursor, Codex, or the Drive UI with your provider key).
 What the agent gets, over MCP:
 
 - **A real desktop**: navigate, snapshot numbered clickable elements, click/fill
-  by ref, exec, files, network capture — no coordinate guessing.
+  by ref, hover menus, upload files under `/home/agent`, marked screenshots,
+  exec, files, network capture — no coordinate guessing. Navigate and click
+  return the first 2000 characters of page text.
 - **Vault logins**: the human saves a credential once (encrypted, via a one-time
   link); the machine types it into the site's own login page. The agent and the
   API never see the password.

SECURITY.md

@@ -9,6 +9,10 @@ Case holds logins. These are promises, with code you can read.
 - **Login only fires** when the page host matches the credential's `domains`.
 - **MCP has no credential-write tool.** Secrets enter via the Drive UI, `/fill`,
   or `bin/case cred add`.
+- **`computer_upload` only assigns files already on the computer.** The path
+  must be under `/home/agent/`, at most 5MB, and the snapshot ref must be
+  `input[type=file]`. Password/OTP-like inputs are refused. Bytes travel
+  through deskd `GET /file`, never command stdout.
 - **cased binds loopback by default.** Compose publishes `127.0.0.1:8787` and
   `127.0.0.1:4174`. Set `CASE_TOKEN` before exposing those ports.
 - **Audit log** (`~/.case/audit/<date>.jsonl`): one line per API call; request

control-plane/cased.py

@@ -318,10 +318,16 @@ def awake(cid, wake):
 
 
 @app.get("/v1/computers/{cid}/screenshot")
-def screenshot(cid: str, wake: bool = False):
+def screenshot(cid: str, wake: bool = False, marks: bool = False):
     with awake(cid, wake) as row:
         # desk_bytes raises ApiError(423) during credential injection
-        return Response(desk_bytes(row, "GET", "/screenshot"), media_type="image/png")
+        content = desk_bytes(row, "GET", "/screenshot")
+        if marks:
+            try:
+                content = browse.overlay_marks(content, browse.element_rects(row))
+            except Exception:
+                pass
+        return Response(content, media_type="image/png")
 
 
 @app.post("/v1/computers/{cid}/action")
@@ -381,6 +387,22 @@ def click_(cid: str, body: dict = Body(...), wake: bool = False):
                                     snapshot_after=bool(body.get("snapshot", True)))
 
 
+@app.post("/v1/computers/{cid}/hover")
+def hover_(cid: str, body: dict = Body(...), wake: bool = False):
+    with awake(cid, wake) as row:
+        if "ref" not in body:
+            raise ApiError(400, "bad_request", "body needs 'ref' (from GET /page)")
+        return browse.hover(row, int(body["ref"]), name=body.get("name"))
+
+
+@app.post("/v1/computers/{cid}/upload")
+def upload_(cid: str, body: dict = Body(...), wake: bool = False):
+    with awake(cid, wake) as row:
+        if "ref" not in body or "path" not in body:
+            raise ApiError(400, "bad_request", "body needs 'ref' and 'path'")
+        return browse.upload(row, int(body["ref"]), body["path"], name=body.get("name"))
+
+
 @app.post("/v1/computers/{cid}/fill")
 def fill_(cid: str, body: dict = Body(...), wake: bool = False):
     with awake(cid, wake) as row:

control-plane/deskclient.py

@@ -71,6 +71,65 @@ def eval_value(row, expression, timeout_s=15, default=None):
 # *existing* container (lifecycle.do_wake), so a new deskd route would reach only
 # computers created after an image rebuild. This reaches every computer that
 # exists today, on a cased restart.
+_PAGE_TEXT = "document.body?document.body.innerText.slice(0,2000):''"
+
+
+def page_text(row, _eval=None):
+    """Best-effort first 2000 chars of the page. None on any failure so a
+    navigation or click that already happened still returns what it did.
+    `_eval` is the eval_js to use — browse patches its own copy."""
+    fn = _eval or eval_js
+    try:
+        r = fn(row, _PAGE_TEXT, 3)
+        t = r.get("value") if r.get("ok") else None
+        return t if isinstance(t, str) and t else None
+    except ApiError:
+        return None
+
+
+def _with_text(row, arrived):
+    t = page_text(row)
+    if t:
+        arrived["text"] = t
+    return arrived
+
+
+_HYDRATE_N = ("document.readyState==='complete'"
+              "?document.querySelectorAll('a,button,input,select,textarea').length:0")
+
+
+def _hydrate(row, arrived, deadline):
+    """SPA: readyState can complete before React mounts any controls."""
+    time.sleep(min(2.0, max(0.0, deadline - time.time())))
+    try:
+        r = eval_js(row, _HYDRATE_N, 3)
+        n = r.get("value") if r.get("ok") else 0
+        if n:
+            return _with_text(row, arrived)
+    except ApiError as e:
+        if e.status != 502:
+            raise
+    if time.time() >= deadline:
+        return _with_text(row, arrived)
+    try:
+        eval_js(row, "location.reload()", 10)
+    except ApiError as e:
+        if e.status != 502:
+            raise
+        return _with_text(row, arrived)
+    while time.time() < deadline:
+        time.sleep(0.4)
+        try:
+            r = eval_js(row, _HYDRATE_N, 3)
+        except ApiError as e:
+            if e.status != 502:
+                raise
+            continue
+        if r.get("ok") and r.get("value"):
+            return _with_text(row, arrived)
+    return _with_text(row, arrived)
+
+
 def navigate(row, url, timeout_s=30):
     """Load `url` in the browser tab and block until the new document is ready.
 
@@ -91,7 +150,8 @@ def navigate(row, url, timeout_s=30):
     """
     deadline = time.time() + timeout_s
     poll = ("window.__case_nav===undefined&&document.readyState==='complete'"
-            "?[location.href,document.title]:null")
+            "?[location.href,document.title,"
+            "document.querySelectorAll('a,button,input,select,textarea').length]:null")
     while True:
         try:
             r = eval_js(row, f"window.__case_nav=1;location.assign({json.dumps(url)})", 10)
@@ -113,8 +173,11 @@ def navigate(row, url, timeout_s=30):
                 raise          # asleep / injecting / wedged, not something to wait out
             continue           # context torn down mid-navigation: that IS progress
         v = r.get("value")
-        if isinstance(v, list):   # a >64KB result comes back as a truncated *string*
-            return {"ok": True, "url": v[0], "title": v[1]}
+        if isinstance(v, list) and len(v) >= 2:   # a >64KB result is a truncated *string*
+            arrived = {"ok": True, "url": v[0], "title": v[1]}
+            if len(v) > 2 and v[2] == 0:
+                return _hydrate(row, arrived, deadline)
+            return _with_text(row, arrived)
     try:   # don't leave a uniquely-named global on a live page for site JS to find
         eval_js(row, "delete window.__case_nav", 3)
     except ApiError:

mcp/case_mcp.py

@@ -78,10 +78,13 @@ def computer_list() -> dict:
 
 
 @mcp.tool()
-def computer_screenshot(computer_id: str) -> Image:
+def computer_screenshot(computer_id: str, marks: bool = False) -> Image:
     """Screenshot of the computer's display (1280x800 by default; see computer_list
-    display for the actual size). Wakes the computer if asleep."""
-    r = call("GET", f"/computers/{computer_id}/screenshot", params={"wake": "true"})
+    display for the actual size). Wakes the computer if asleep.
+    marks=true draws numbered boxes matching snapshot refs (drawn on the PNG in
+    the control plane — the live page is not modified)."""
+    r = call("GET", f"/computers/{computer_id}/screenshot",
+             params={"wake": "true", "marks": "true" if marks else "false"})
     return Image(data=r.content, format="png")
 
 
@@ -145,12 +148,12 @@ def computer_eval(computer_id: str, expression: str, timeout_s: int = 20) -> dic
 def computer_navigate(computer_id: str, url: str, timeout_s: int = 30) -> dict:
     """Point the computer's browser at url and block until the page has loaded.
     One call — do not follow it with readyState polling. Returns
-    {ok, url, title, snapshot} (url is the final one, after redirects) or
-    {ok:false, error}. `snapshot` is the arrived page's numbered elements, already
-    there — DO NOT call computer_snapshot after this, you have it.
+    {ok, url, title, text, snapshot} (url is the final one, after redirects) or
+    {ok:false, error}. `text` is the first 2000 chars of the page; only eval
+    innerText when you need more. `snapshot` is the arrived page's numbered
+    elements — DO NOT call computer_snapshot after this, you have it.
     The body is ready when this returns; `title` is best-effort and can be "" on
     pages that set it a beat late — that is not a signal to wait or retry.
-    Read page text with computer_eval("document.body.innerText").
     Same-page '#anchor' jumps are not navigations; use computer_eval for those."""
     return call("POST", f"/computers/{computer_id}/navigate", params={"wake": "true"},
                 json={"url": url, "timeout_s": timeout_s},
@@ -165,12 +168,10 @@ def computer_snapshot(computer_id: str) -> dict:
     coordinate guessing. Returns {ok, url, title, count, elements} where each
     element is a line like '[12] button "Save changes"' or
     '[13] input(email) "" =\\'\\' — pass that number to computer_click_element or
-    computer_fill. Everything currently on screen is included; on a big page the rest
-    is cut to a budget, so `count` can exceed the lines you get and the numbers SKIP —
-    that is normal, use the number on the line, never its position in the list. Scroll
-    and snapshot again to reach what was cut.
-    Refs are re-derived per call (document order), so a ref is valid
-    until the page changes. You rarely need this tool twice: computer_navigate,
+    computer_fill. Starred lines (`*[n]`) appeared since the last snapshot
+    (autocomplete/typeahead); click one, do not press Enter. Refs are
+    re-derived per call (document order), so a ref is valid until the page
+    changes. You rarely need this tool twice: computer_navigate,
     computer_click_element and computer_fill(submit) all return the fresh snapshot
     in their own result. Call this for the FIRST look at a page, or after something
     changed it that Case did not do (a timer, a redirect you waited out).
@@ -184,14 +185,16 @@ def computer_click_element(computer_id: str, ref: int, name: str = None,
                            text: str = None, screenshot: bool = False) -> dict:
     """Click element [ref] from the last computer_snapshot. Pass name (the quoted
     text from the snapshot line) so a changed page is caught: on mismatch this
-    REFUSES to click and returns {ok:false, stale:true, snapshot} — use the fresh
-    snapshot and retry with the right ref; never click blind after a refusal.
+    REFUSES to click and returns {ok:false, stale:true, snapshot} unless exactly
+    one current element matches name — then it heals and clicks ({healed:true}).
+    Use the fresh snapshot after a refusal; never click blind.
     The element is scrolled into view and clicked with a real OS-level mouse event
-    (isTrusted true). text, when given, is typed into the element after the click
+    (isTrusted true). A click that opens a new tab activates it and returns
+    switched_tab. Returns the first 2000 chars of page text after the click.
+    text, when given, is typed into the element after the click
     (click focuses it) — for one field that beats computer_fill.
-    On success the result carries `snapshot`: the page as it stands after the click,
-    settled. DO NOT call computer_snapshot after this — read the refs from there and
-    click again. One call is the whole act-then-look loop."""
+    On success the result also carries `snapshot`: the page as it stands after the
+    click, settled. DO NOT call computer_snapshot after this."""
     body = {"ref": ref}
     if name is not None:
         body["name"] = name
@@ -203,6 +206,29 @@ def computer_click_element(computer_id: str, ref: int, name: str = None,
                 params={"wake": "true"}, json=body, timeout=60).json()
 
 
+@mcp.tool()
+def computer_hover(computer_id: str, ref: int, name: str = None) -> dict:
+    """Hover the OS pointer over snapshot [ref] without clicking — opens menus
+    that only appear on hover. Pass name so a changed page is refused."""
+    body = {"ref": ref}
+    if name is not None:
+        body["name"] = name
+    return call("POST", f"/computers/{computer_id}/hover",
+                params={"wake": "true"}, json=body, timeout=40).json()
+
+
+@mcp.tool()
+def computer_upload(computer_id: str, ref: int, path: str, name: str = None) -> dict:
+    """Assign a file already on the computer (path under /home/agent/, ≤5MB) to
+    snapshot [ref], which must be input[type=file]. Never send file bytes through
+    this tool — write the file with computer_file_put or computer_exec first."""
+    body = {"ref": ref, "path": path}
+    if name is not None:
+        body["name"] = name
+    return call("POST", f"/computers/{computer_id}/upload",
+                params={"wake": "true"}, json=body, timeout=90).json()
+
+
 @mcp.tool()
 def computer_fill(computer_id: str, fields: list, submit: bool = False) -> dict:
     """Fill a whole form in ONE call: fields=[{"ref": 13, "value": "jane@x.com"}, …]

requirements.txt

@@ -5,3 +5,4 @@ docker==7.1.0
 cryptography==49.0.0
 requests==2.34.2
 mcp==1.28.1
+Pillow==12.2.0