feat(harness): lease a pool sandbox for prompt leaves

deploy/knative/demo-remote-worker.sh

@@ -48,9 +48,10 @@ SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")"
 cd "$(dirname "$0")"
 source ./lib.sh   # NS, KSVC, BASE, CURL_OPTS, CURL_HDR, ok/ko, PASS/FAIL, ensure_port_forward
 # shellcheck source=./lib-relay.sh
-source ./lib-relay.sh  # MODEL, claim/abort, dispatch_pattern, assert_verdict,
-                       # validate_discriminator, assert_presence, assert_no_pods_match,
-                       # snapshot/flip/restore_harness_env. Shared with relay-leaf-smoke.sh.
+source ./lib-relay.sh  # MODEL, claim/abort, dispatch_prompt, assert_reply_contains/_lacks,
+                       # reply_text, validate_discriminator, assert_presence(_gone),
+                       # assert_no_pods_match, snapshot/flip/restore_harness_env.
+                       # Shared with relay-leaf-smoke.sh (which keeps the verdict pair).
 
 REPO_ROOT="$(cd ../.. && pwd)"
 CLUSTER_NAME="${CLUSTER_NAME:-sh-knative}"
@@ -72,6 +73,11 @@ RELAY_TOKEN="${SANDBOX_TOKEN:-}"
 IN_CLUSTER_RELAY_ADDR="sandbox-relay.${NS}.svc:8443"
 # A label no pod carries, so the remote worker is the only lease candidate (defense 1).
 REMOTE_ONLY_SELECTOR="sh.kagenti.io/sandbox-pool=demo-remote-only"
+# The free-form ask, identical for both backends -- the reply NAMES the OS it read, so which
+# filesystem answered is stated by the model rather than inferred from a flag. "read tool" and
+# the absolute path are load-bearing: the tools run in the sandbox, and a relative path would be
+# resolved against the harness process cwd instead (cf. buildLeafPrompt in harness/src/run-leaf.ts).
+OS_PROMPT="Using your read tool, read the file /etc/os-release and tell me in one sentence exactly which OS distribution and version it reports."
 PF_LOG="${TMPDIR:-/tmp}/sh-demo-relay-pf.$$.log"
 
 REUSE_CLUSTER=0
@@ -448,18 +454,20 @@ claim "Discriminator: verify Alpine vs RHEL BEFORE relying on it"
 POD_OS="$(kubectl exec "$SBOX_POD" -n "$NS" -- cat /etc/os-release 2>/dev/null || true)"
 WORKER_OS="$(docker exec "$WORKER_CTR" cat /etc/os-release 2>/dev/null || true)"
 validate_discriminator "$POD_OS" "$WORKER_OS" "$SBOX_POD" "$WORKER_CTR (host container)"
-note "Now a verdict on the pattern 'Alpine' identifies which filesystem answered."
+note "Now a free-form prompt asking what /etc/os-release says identifies which filesystem answered."
 
 # --- 9. Run A: the pod path ------------------------------------------------------------
 claim "Run A -- in-cluster Alpine sandbox pod"
 [ "$(ksvc_env_value SH_REMOTE_SANDBOX)" = "1" ] \
   && abort "the harness is already on SH_REMOTE_SANDBOX=1, so run A would not be the pod path. Run '$0 --teardown' first, or restore the ksvc env by hand."
 snapshot_harness_env
-RESP_A="$(dispatch_pattern "demo-pod-$$" "Alpine")"
-assert_verdict "A/pod-path/Alpine" "$RESP_A" "FLAGGED" \
-  "the in-cluster pool is Alpine, so a leaf grepping for 'Alpine' must be FLAGGED here; an empty verdict usually means the model endpoint is unreachable"
-REASON_A="$(verdict_reason "$RESP_A")"
-[ -n "$REASON_A" ] && note "model's reason: $REASON_A"
+RESP_A="$(dispatch_prompt "demo-pod-$$" "$OS_PROMPT")"
+assert_reply_contains "A/pod-path" "$RESP_A" "Alpine" \
+  "the in-cluster pool is Alpine, so the reply must name Alpine here; an empty reply usually means the model endpoint is unreachable"
+assert_reply_lacks "A/pod-path" "$RESP_A" "Red Hat" \
+  "naming Red Hat here means the exec already went remote, so run A is not the pod baseline it claims to be"
+TEXT_A="$(reply_text "$RESP_A")"
+[ -n "$TEXT_A" ] && note "model's reply: $TEXT_A"
 
 # --- 10. Flip --------------------------------------------------------------------------
 claim "Flip to the remote path (and make a pod win IMPOSSIBLE)"
@@ -474,28 +482,63 @@ note "worker -> relay via host.docker.internal:${RELAY_PORT} (outbound through t
 
 # --- 11. Run B: the remote path --------------------------------------------------------
 claim "Run B -- remote host container (both directions asserted)"
-RESP_B="$(dispatch_pattern "demo-remote-alpine-$$" "Alpine")"
-assert_verdict "B/remote/Alpine" "$RESP_B" "CLEAR" \
-  "FLAGGED here means the exec landed on an Alpine sandbox pod, not the remote RHEL worker -- the demo would be proving nothing"
-REASON_B="$(verdict_reason "$RESP_B")"
-[ -n "$REASON_B" ] && note "model's reason: $REASON_B"
-RESP_B2="$(dispatch_pattern "demo-remote-redhat-$$" "Red Hat")"
-assert_verdict "B/remote/RedHat" "$RESP_B2" "FLAGGED" \
-  "CLEAR here means the exec landed on an Alpine sandbox pod, not the remote RHEL worker -- the demo would be proving nothing"
+RESP_B="$(dispatch_prompt "demo-remote-$$" "$OS_PROMPT")"
+assert_reply_contains "B/remote" "$RESP_B" "Red Hat" \
+  "the reply must name Red Hat: anything else means the exec landed on an Alpine sandbox pod, not the remote RHEL worker -- the demo would be proving nothing"
+assert_reply_lacks "B/remote" "$RESP_B" "Alpine" \
+  "naming Alpine means the exec landed on an in-cluster pod despite the remote-only selector"
+TEXT_B="$(reply_text "$RESP_B")"
+[ -n "$TEXT_B" ] && note "model's reply: $TEXT_B"
 note "Asserted both ways: a pod-landed exec fails one check or the other, never neither."
 
-# --- 12. Summary -----------------------------------------------------------------------
+# --- 12. Planted marker: evidence that exists ONLY on this laptop ----------------------
+# The strongest assertion in the demo, and the one a free-form reply carries best: /etc/os-release
+# can be argued with (image drift, a cached answer, a model that guesses "Red Hat" from context),
+# but a random string written seconds ago into a container the cluster cannot reach cannot be.
+claim "Planted marker -- the cluster reads a secret created on this laptop"
+MARK="tuscan-lentils-$RANDOM-$$"
+docker exec "$WORKER_CTR" sh -c "echo 'secret marker: $MARK' > /tmp/proof.txt" \
+  || abort "could not write the marker into $WORKER_CTR"
+# Fail closed: if the pod DOES have the file, the discriminator is void and the next assertion
+# would pass for the wrong reason.
+if kubectl exec "$SBOX_POD" -n "$NS" -- cat /tmp/proof.txt >/dev/null 2>&1; then
+  abort "/tmp/proof.txt unexpectedly exists on $SBOX_POD -- the marker no longer distinguishes the two filesystems"
+fi
+ok "marker exists only in $WORKER_CTR on this host; $SBOX_POD has no /tmp/proof.txt"
+RESP_C="$(dispatch_prompt "demo-proof-$$" "Using your read tool, read the file /tmp/proof.txt and tell me exactly what marker string it contains.")"
+assert_reply_contains "C/remote/marker" "$RESP_C" "$MARK" \
+  "the cluster did not read the planted marker -- either the exec did not reach the remote worker, or it read a different filesystem"
+TEXT_C="$(reply_text "$RESP_C")"
+[ -n "$TEXT_C" ] && note "model's reply: $TEXT_C"
+
+# --- 13. Presence vanishes with the stream ---------------------------------------------
+# Skipped under --keep, which promises the worker is still running when the run ends: proving the
+# record clears means closing the Attach stream, and there is no way to do both.
+if [ "$KEEP" = 1 ]; then
+  claim "Presence teardown -- SKIPPED (--keep)"
+  note "--keep leaves the worker running, so its Attach stream stays open and the record stays."
+  note "Re-run without --keep to assert the record clears, or watch it go by hand:"
+  note "  docker stop $WORKER_CTR"
+  note "  kubectl exec deploy/redis -n $NS -- redis-cli HGETALL sh:sandbox:records"
+else
+  claim "Presence: the record clears when the Attach stream closes"
+  docker stop "$WORKER_CTR" >/dev/null 2>&1 || true
+  assert_presence_gone "$SANDBOX_ID"
+  note "Nothing deleted it -- the registration IS the stream, so it went with it."
+fi
+
+# --- 14. Summary -----------------------------------------------------------------------
 echo ""
 echo "=== The A/B ==="
-printf '%-34s | %-8s | %s\n' "backend" "'Alpine'" "model's stated reason"
-printf '%-34s-+-%-8s-+-%s\n' "----------------------------------" "--------" "---------------------"
-printf '%-34s | %-8s | %s\n' "in-cluster pod ($SBOX_POD)" \
-  "$(jq -r '.verdict.verdict // "?"' <<<"$RESP_A" 2>/dev/null || echo '?')" "${REASON_A:-(none returned)}"
-printf '%-34s | %-8s | %s\n' "remote host container" \
-  "$(jq -r '.verdict.verdict // "?"' <<<"$RESP_B" 2>/dev/null || echo '?')" "${REASON_B:-(none returned)}"
+printf '%-34s | %s\n' "backend" "what the model said it read"
+printf '%-34s-+-%s\n' "----------------------------------" "---------------------------"
+printf '%-34s | %s\n' "in-cluster pod ($SBOX_POD)" "${TEXT_A:-(no reply returned)}"
+printf '%-34s | %s\n' "remote host container" "${TEXT_B:-(no reply returned)}"
 echo ""
-echo "Same request, opposite verdicts. The second one ran on a container on this host that"
-echo "the cluster cannot reach and never authenticated to -- it dialed out, and nothing else."
+echo "Same prompt, and the model names a different OS each time. The second one ran on a"
+echo "container on this host that the cluster cannot reach and never authenticated to -- it"
+echo "dialed out, and nothing else. Then it read a marker planted here seconds earlier:"
+printf '  %s\n' "${MARK}"
 
 echo ""; echo "=== Results: $PASS passed, $FAIL failed ==="
 if [ "$FAIL" -gt 0 ]; then echo "DEMO FAIL"; exit 1; else echo "DEMO PASS"; exit 0; fi

deploy/knative/lib-relay.sh

@@ -224,6 +224,69 @@ verdict_reason() {
   jq -r '.verdict.reason // empty' <<<"$1" 2>/dev/null || true
 }
 
+# --- Prompt dispatch + reply assertions ------------------------------------------------
+# The free-form counterpart to dispatch_pattern/assert_verdict above. Both pairs are kept:
+# the demo proves placement with a prompt whose reply NAMES the OS it read, while
+# relay-leaf-smoke.sh keeps the verdict pair, whose binary CLEAR/FLAGGED is the stronger
+# gate for a live smoke that cannot run in CI.
+
+# dispatch_prompt <sessionId> <prompt> -> echoes terminal JSON from POST /runs for a
+# kind:prompt leaf. The reply lands in .text as the model's own words.
+# `|| true` for the same reason as dispatch_pattern: a connection-level failure must yield an
+# empty body so the assertion's "endpoint unreachable" hint is reached, not bypassed by set -e.
+dispatch_prompt() {
+  local sid="$1" prompt="$2" body
+  body=$(jq -nc --arg s "$sid" --arg m "$MODEL" --arg p "$prompt" \
+    '{sessionId:$s, model:$m, kind:"prompt", prompt:$p}')
+  # shellcheck disable=SC2086  # CURL_OPTS is intentionally word-split
+  curl -s $CURL_OPTS --max-time 120 ${CURL_HDR[@]+"${CURL_HDR[@]}"} \
+    -H "Content-Type: application/json" -d "$body" "$BASE/runs" || true
+}
+
+# Echo the model's reply text (empty when absent). Usage: reply_text <response-json>
+reply_text() {
+  jq -r '.text // empty' <<<"$1" 2>/dev/null || true
+}
+
+# Shared precondition for the reply assertions: echo the reply text, or report the
+# "no reply at all" failure and return non-zero. Kept in one place so a missing reply can
+# never be mistaken for a reply that merely lacks the needle.
+_reply_or_ko() {
+  local label="$1" resp="$2" text
+  text="$(reply_text "$resp")"
+  if [ -z "$text" ]; then
+    # Two very different causes look identical here -- name both, as assert_verdict does.
+    ko "$label: no reply text returned (empty/non-JSON response -- either the harness endpoint at $BASE is unreachable, or the model is; check the kourier port-forward, then the llm-credentials secret / SH_MODEL); raw: $(echo "$resp" | head -c 200)"
+    return 1
+  fi
+  echo "$text"
+}
+
+# assert_reply_contains <label> <response-json> <needle> <hint-if-absent>
+# Case-insensitive substring match on the model's reply.
+assert_reply_contains() {
+  local label="$1" resp="$2" needle="$3" hint="$4" text
+  text="$(_reply_or_ko "$label" "$resp")" || return
+  if grep -qiF -- "$needle" <<<"$text"; then
+    ok "$label: reply names '$needle'"
+  else
+    ko "$label: reply does not name '$needle' -- $hint; reply: $(echo "$text" | head -c 300)"
+  fi
+}
+
+# assert_reply_lacks <label> <response-json> <needle> <hint-if-present>
+# The other half of the placement proof: a free-form reply has no binary flag to flip, so the
+# wrong backend is ruled out by asserting the OS it must NOT have read is absent as well.
+assert_reply_lacks() {
+  local label="$1" resp="$2" needle="$3" hint="$4" text
+  text="$(_reply_or_ko "$label" "$resp")" || return
+  if grep -qiF -- "$needle" <<<"$text"; then
+    ko "$label: reply names '$needle' but must not -- $hint; reply: $(echo "$text" | head -c 300)"
+  else
+    ok "$label: reply does not name '$needle'"
+  fi
+}
+
 # --- Harness env flip / restore --------------------------------------------------------
 # Capture the harness ksvc env exactly, for exact restore later.
 snapshot_harness_env() {

deploy/knative/tests/lib-relay-shared.test.sh

@@ -2,10 +2,10 @@
 # deploy/knative/tests/lib-relay-shared.test.sh
 #
 # Locks the property lib-relay.sh exists to provide: the assertions that decide whether a
-# remote-sandbox proof HOLDS are defined once and shared by both callers. Nothing else
-# enforces it -- a future edit can re-inline a copy of validate_discriminator or
-# assert_verdict into either script and every existing check still passes, while the two
-# proofs quietly drift apart and one keeps asserting something the other dropped.
+# remote-sandbox proof HOLDS are defined once in lib-relay.sh. Nothing else enforces it -- a
+# future edit can re-inline a copy of validate_discriminator or assert_verdict into either
+# script and every existing check still passes, while the two proofs quietly drift apart and
+# one keeps asserting something the other dropped.
 #
 # Static: greps the scripts, runs no cluster and no kubectl. Complements the live gate
 # (RELAY_LIVE_SMOKE=1 relay-leaf-smoke.sh), which cannot run in CI.
@@ -17,8 +17,21 @@ LIB="$DIR/lib-relay.sh"
 CALLERS=("$DIR/relay-leaf-smoke.sh" "$DIR/demo-remote-worker.sh")
 
 # The assertions whose duplication would let the two proofs disagree about what "passing"
-# means. Not every helper in lib-relay.sh -- only the ones that decide a verdict.
-SHARED_FNS=(validate_discriminator assert_verdict dispatch_pattern assert_no_pods_match count_pool_pods)
+# means. Not every helper in lib-relay.sh -- only the ones that decide an outcome.
+#
+# The two callers now dispatch differently ON PURPOSE, and both pairs are locked here:
+#   - demo-remote-worker.sh uses dispatch_prompt + assert_reply_contains/_lacks, whose reply
+#     NAMES the OS the model read -- that is what the demo has to show a room.
+#   - relay-leaf-smoke.sh keeps dispatch_pattern + assert_verdict, whose binary CLEAR/FLAGGED
+#     is the stronger gate for a live smoke that cannot run in CI.
+# What must NOT diverge is the placement machinery both proofs rest on
+# (validate_discriminator, assert_no_pods_match, count_pool_pods) -- and neither pair may be
+# re-inlined into a caller, or the two proofs can drift apart exactly as before.
+SHARED_FNS=(
+  validate_discriminator assert_no_pods_match count_pool_pods
+  assert_verdict dispatch_pattern
+  dispatch_prompt assert_reply_contains assert_reply_lacks reply_text
+)
 
 FAILS=0
 pass() { echo "  ok: $1"; }

docs/adrs/0028-async-prompt-dispatch.md

@@ -1,6 +1,6 @@
 # ADR-0028: Async prompt dispatch as a `kind:"prompt"` leaf sharing the `/turn` core
 
-- **Status:** Proposed <!-- Proposed → Accepted → Superseded by ADR-NNNN / Deprecated -->
+- **Status:** Proposed, amended 2026-09-01 (see [Amendment](#amendment-2026-09-01-prompt-leaves-lease-a-pool-sandbox)) <!-- Proposed → Accepted → Superseded by ADR-NNNN / Deprecated -->
 - **Date:** 2026-08-25
 - **Deciders:** Serverless Harness team
 - **Spec:** [`../specs/2026-08-25-async-prompt-dispatch-design.md`](../specs/2026-08-25-async-prompt-dispatch-design.md)
@@ -41,6 +41,43 @@ default`) and inherits `/turn`'s `resolveSandboxConfig` sandbox routing.
 - Negative / accepted cost: prompt leaves get no per-leaf pool isolation (they share `/turn`'s sandbox model), so a fleet of async prompts is not lease-bounded the way solve leaves are; `runTurn` is refactored, so its behavior is now pinned by a regression test rather than by being the only caller. A prompt leaf may still be addressed to a `workloadId` (the workload gates existence and returns 404 if absent), but its pool selector is intentionally ignored — the API boundary logs a warning rather than injecting a selector that `executeTurn` would silently drop.
 - Follow-up owed: pool-based isolation (a `selectPoolSandbox` lease) for prompt leaves, deferred until a driver needs it; extend `deploy/knative/leaf-async-smoke.sh` with a `responded` claim.
 
+## Amendment (2026-09-01): prompt leaves lease a pool sandbox
+
+The deferred follow-up above now has its driver, and the deferral turned out to cost more than
+"no per-leaf isolation". The remote-sandbox demo
+([`../demos/remote-sandbox-demo.md`](../demos/remote-sandbox-demo.md)) needs a free-form prompt to
+run its tool calls on a laptop container reached over the sandbox relay, and
+`SH_REMOTE_SANDBOX`/`SH_RELAY_ADDR` are read only on the lease path — so a prompt leaf could not
+reach a remote sandbox at all. Worse, on any deployment that configures a pool the way
+`deploy/knative/service.yaml` does (a `KAGENTI_SANDBOX_POOL_SELECTOR`, no `KAGENTI_SANDBOX_POD`),
+`resolveSandboxConfig` returned null and a prompt leaf's tool calls ran **in the harness container's
+own filesystem** — silently, and indistinguishably from a sandbox that happened to answer.
+
+**Amended decision.** `runPromptLeaf` leases through `selectPoolSandbox` exactly as the converge and
+solve paths do — heartbeat while the turn runs, release in a `finally` — and hands the leased
+sandbox to the turn. `executeTurn` gains an optional pre-leased `sandbox`; `resolveTurnSandbox`
+returns it verbatim when present and otherwise falls back to `resolveSandboxConfig`, so **`/turn`
+is unchanged**. This is a superset rather than a swap: with no pool selector configured
+`selectPoolSandbox` performs that same single-pod resolution itself.
+
+The alternative rejected above ("imports the saturation/503 path") is accepted as a consequence
+rather than avoided: saturation propagates as `{status:"failed", reason:"saturated"}`, which the
+sync `/runs` path bounded-waits then 503s on, and which `classifyOutcome` already keeps retryable
+for the async queue. Suppressing it would have taken extra code to be less consistent. The
+"identical sync or async" promise is preserved — both paths run the same `runPromptLeaf` — but a
+prompt leaf can now 503 on a saturated pool, where before it could not.
+
+A second, smaller contract change falls out of the `finally` that releases the lease: a throw from
+the turn is now caught and returned as `{status:"failed", reason:"error"}` instead of propagating
+out of `runLeaf` (which the sync route turned into a `500`). That matches what the converge and
+solve paths have always done, and `classifyOutcome` already treats `error` as retryable, so the
+async queue behaves the same as before.
+
+- Still deferred: a **workload-addressed** prompt leaf (`workloadId`) continues to ignore the
+  workload's own `sandboxSelector`, and the API boundary still logs that warning. The envelope's
+  `sandboxPoolSelector` is honored now, so only the workload-resolver special case remains, and
+  whether a workload's pool should bound its prompt leaves is a separate call from this one.
+
 ---
 
 *Assisted-By: Claude (Anthropic AI) <noreply@anthropic.com>*