diff --git a/Makefile b/Makefile index 6e79da3..b76823d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ -.PHONY: lint fmt test test-deploy typecheck demo-remote-sandbox demo-remote-sandbox-teardown +.PHONY: lint fmt test test-deploy typecheck demo-remote-sandbox demo-remote-sandbox-teardown \ + demo-promoted-workflow demo-promoted-workflow-teardown lint: pre-commit run --all-files @@ -31,3 +32,12 @@ demo-remote-sandbox: demo-remote-sandbox-teardown: bash deploy/knative/demo-remote-worker.sh --teardown + +# Promote a Claude Code workflow authored in a minimal local sandbox, then prove it ran remotely. +# Needs a warm cluster whose image contains the promotion feature; the script gates on that. +# See docs/demos/promoted-workflow-demo.md. +demo-promoted-workflow: + bash deploy/knative/demo-promoted-workflow.sh $(DEMO_ARGS) + +demo-promoted-workflow-teardown: + bash deploy/knative/demo-promoted-workflow.sh --teardown diff --git a/deploy/knative/demo-promoted-workflow.sh b/deploy/knative/demo-promoted-workflow.sh new file mode 100644 index 0000000..3fd6816 --- /dev/null +++ b/deploy/knative/demo-promoted-workflow.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +# deploy/knative/demo-promoted-workflow.sh +# Scripted sibling of docs/demos/promoted-workflow-demo.md — asserts every claim the guided +# walkthrough makes, with no narration. +# +# The claim: a Claude Code workflow authored in a MINIMAL LOCAL SANDBOX (one skill, a CLAUDE.md, +# one memory file, one slash command) runs unchanged in the harness. Proved by dispatching the +# SAME prompt twice to the SAME cluster, differing only by the `configRef` field: +# +# A (bare) -> generic prose. Cannot cite KAG-4471; nothing told it that id exists. +# B (promoted) -> cites KAG-4471 (promoted memory), carries a RISK line (promoted CLAUDE.md), +# and echoes a token readable ONLY inside the sandbox pod (promoted skill). +# +# Why the token is the load-bearing claim: pi puts a skill's name/description in the system prompt +# and tells the model to `read` the body on demand (pi-fork core/skills.ts formatSkillsForPrompt). +# That read executes in the SEPARATE sandbox pod. So the token cannot appear unless the bundle +# materialised on BOTH halves of the fs-free split and the injected absolute skills path resolved +# to the skill's own subdirectory. It is the sibling-read claim of the spec's live smoke, dispatched +# over HTTP instead of in-process. +# +# The `HOME` override is the point of the demo, not a trick to skip work: it makes promote read the +# sandbox as USER scope, so exactly one skill travels instead of a whole ~/.claude. That is the +# sandbox-first authoring the design recommends (spec §11). +# +# Prereqs: a warm harness cluster (setup-kind.sh) whose image CONTAINS the promotion feature. +# The capability gate below refuses to run against an older image rather than print a green +# run that proves nothing. jq, curl, kubectl, pnpm and a built pi-fork are required. +# Usage: +# bash deploy/knative/demo-promoted-workflow.sh +# bash deploy/knative/demo-promoted-workflow.sh --keep-sandbox # leave /tmp/sh-demo in place +# bash deploy/knative/demo-promoted-workflow.sh --teardown # remove the sandbox and exit +set -euo pipefail +# Resolve $0 BEFORE the cd: --help re-reads this file, and a relative argv[0] (the usual +# `bash deploy/knative/demo-promoted-workflow.sh --help`) stops resolving once the cwd moves. +SELF="$(cd "$(dirname "$0")" && pwd)/$(basename "$0")" +cd "$(dirname "$0")" +source ./lib.sh # NS, BASE, HOST_HEADER, CURL_HDR, CURL_OPTS, ok/ko, PASS/FAIL + +REPO_ROOT="$(cd ../.. && pwd)" +FIXTURE="$(pwd)/fixtures/promoted-demo" +SANDBOX="${SH_DEMO_SANDBOX:-/tmp/sh-demo}" +# Marker file gating every destructive touch of $SANDBOX. Without it this script would happily +# rm -rf a directory a caller pointed SH_DEMO_SANDBOX at by mistake. +MARKER=".sh-demo-sandbox" +MODEL="${SH_MODEL:-claude-haiku-4-5}" +REDIS_PORT="${SH_DEMO_REDIS_PORT:-16379}" +KEEP_SANDBOX=0 +TEARDOWN=0 +# The unguessable fact: it exists only in the promoted memory file, so a bare run cannot cite it. +TICKET="KAG-4471" +TOKEN="SHIPNOTE-7F3A-SANDBOX-OK" +PROMPT="Write the ship note for the auth timeout fix." + +for a in "$@"; do + case "$a" in + --keep-sandbox) KEEP_SANDBOX=1 ;; + --teardown) TEARDOWN=1 ;; + -h | --help) + sed -n '2,31p' "$SELF" # 31 is the last comment line; 32 is `set -euo pipefail` + exit 0 + ;; + *) + echo "unknown flag: $a" >&2 + exit 2 + ;; + esac +done + +PFS=() # port-forward pids we started; only ours are killed on exit +# Initialised before cleanup() can run: it is the EXIT trap, and `set -u` would abort on an unset +# name if a claim failed before these were assigned. +promote_log="" +repromote_log="" +cleanup() { + for pid in ${PFS[@]+"${PFS[@]}"}; do kill "$pid" 2>/dev/null || true; done + rm -f "$promote_log" "$repromote_log" 2>/dev/null || true + if [ "$KEEP_SANDBOX" -eq 0 ] && [ -f "$SANDBOX/$MARKER" ]; then rm -rf "$SANDBOX"; fi +} +trap cleanup EXIT + +reset_sandbox() { + if [ -e "$SANDBOX" ] && [ ! -f "$SANDBOX/$MARKER" ]; then + echo "REFUSING to touch $SANDBOX: it exists and holds no $MARKER marker." >&2 + echo "Point SH_DEMO_SANDBOX at a fresh path, or delete that directory yourself." >&2 + exit 2 + fi + rm -rf "$SANDBOX" + mkdir -p "$SANDBOX/.claude" + touch "$SANDBOX/$MARKER" +} + +# Claude Code's own project slug: every path separator becomes '-'. promote.ts:projectMemoryDir +# mirrors this to FIND the memory directory Claude Code already made, so the demo must produce the +# same shape rather than a tidier one. +memory_dir_for() { echo "$SANDBOX/.claude/projects/-$(echo "${1#/}" | tr '/' '-')/memory"; } + +claim() { + echo "" + echo "--- Claim $1: $2 ---" +} + +# --------------------------------------------------------------------------------------------- +if [ "$TEARDOWN" -eq 1 ]; then + if [ -f "$SANDBOX/$MARKER" ]; then + rm -rf "$SANDBOX" + echo "removed $SANDBOX" + else + echo "nothing to remove at $SANDBOX (no $MARKER marker)" + fi + KEEP_SANDBOX=1 # cleanup() must not re-run the removal + exit 0 +fi + +echo "=== Promoted-workflow demo (model=$MODEL, sandbox=$SANDBOX) ===" + +# --- Step 0: dependencies and a warm cluster ------------------------------------------------- +for bin in jq curl kubectl pnpm; do + command -v "$bin" >/dev/null 2>&1 || { + echo "MISSING dependency: $bin" >&2 + exit 2 + } +done + +kubectl -n "$NS" get pod -l app=redis -o name >/dev/null 2>&1 || + kubectl -n "$NS" get deploy/redis >/dev/null 2>&1 || { + echo "no redis in namespace $NS — run ./deploy/knative/setup-kind.sh first" >&2 + exit 2 + } +kubectl wait ksvc/"$KSVC" -n "$NS" --for=condition=Ready --timeout=120s >/dev/null || { + echo "ksvc $KSVC is not Ready" >&2 + exit 2 +} + +# Kourier port-forward for dispatch; skipped in Route mode (KSVC_URL set). +if [ -z "${KSVC_URL:-}" ] && ! curl -s $CURL_OPTS -o /dev/null --max-time 2 "$BASE/" 2>/dev/null; then + kubectl port-forward -n kourier-system svc/kourier "${PORT}:80" >/dev/null 2>&1 & + PFS+=($!) + sleep 3 +fi + +# Redis port-forward so `promote` uploads into the CLUSTER's store. +# +# Deliberately NOT 6379, and deliberately no "something already listens, reuse it" shortcut. A +# listener on 6379 is not evidence that it is this cluster's Redis: a local test container +# (`sh-tdd-redis` publishes 0.0.0.0:6379) answers the probe just as happily, and then promote +# uploads into it while the harness reads the cluster's — reporting a successful upload and +# `config bundle not found` for the very digest it just wrote. Measured, not hypothetical: it cost +# this demo two model calls before Claim 3 failed with the digest right there in the message. +# Bind our own private port instead, and fail loudly if it is taken. +if (exec 3<>/dev/tcp/127.0.0.1/"$REDIS_PORT") 2>/dev/null; then + exec 3>&- 2>/dev/null || true + echo "port $REDIS_PORT is already in use; set SH_DEMO_REDIS_PORT to a free port" >&2 + exit 2 +fi +kubectl port-forward -n "$NS" svc/redis "${REDIS_PORT}:6379" >/dev/null 2>&1 & +PFS+=($!) +sleep 3 +export REDIS_URL="redis://localhost:${REDIS_PORT}" + +dispatch() { # dispatch [configRef] + local sid="$1" prompt="$2" ref="${3:-}" body + if [ -n "$ref" ]; then + body=$(jq -nc --arg s "$sid" --arg m "$MODEL" --arg p "$prompt" --arg c "$ref" \ + '{sessionId:$s, kind:"prompt", model:$m, prompt:$p, configRef:$c}') + else + body=$(jq -nc --arg s "$sid" --arg m "$MODEL" --arg p "$prompt" \ + '{sessionId:$s, kind:"prompt", model:$m, prompt:$p}') + fi + curl -s $CURL_OPTS --max-time 240 ${CURL_HDR[@]+"${CURL_HDR[@]}"} \ + -H "Content-Type: application/json" -d "$body" "$BASE/runs" +} + +# --- Claim 0: the deployed image actually implements promotion -------------------------------- +# An unknown digest MUST fail before any model call. An image without the feature ignores the +# field and answers normally -- which would make Claim 3's A/B look like a model mood swing +# instead of a missing deployment. Gate on it. +claim 0 "the deployed harness resolves configRef (an unknown digest fails loudly)" +probe=$(dispatch "demo-cfg-probe-$$" "say hi" "sha256:$(printf 'f%.0s' {1..64})") +if [ "$(jq -r '.status' <<< "$probe")" = "failed" ]; then + ok "unknown digest rejected: $(jq -r '.message' <<< "$probe" | head -c 60)" +else + ko "the deployed image IGNORED configRef — it predates the promotion feature" + cat >&2 < "$promote_log" 2>&1; then + ko "promote failed" + cat "$promote_log" >&2 + exit 1 +fi +sed -n '/^project:/,$p' "$promote_log" + +travels=$(grep -E '^ travels' "$promote_log" | awk '{print $2}') +[ "$travels" = "1" ] && ok "exactly 1 skill travels" || ko "expected 1 travelling skill, got '$travels'" +grep -q '^preflight: no findings' "$promote_log" && + ok "zero preflight findings" || ko "preflight reported findings (see log above)" + +DIGEST=$(grep -oE 'sha256:[0-9a-f]{64}' "$promote_log" | head -1) +[ -n "$DIGEST" ] || { + ko "no digest in promote output" + exit 1 +} +echo " digest: $DIGEST" +[ -f "$SANDBOX/.claude/promoted.lock.json" ] && + ok "lockfile written" || ko "no .claude/promoted.lock.json" + +# Read the key back through the CLUSTER's own client, not through our forwarded socket. A +# port-forward that reached the wrong Redis passes every check above -- promote prints "uploaded" +# either way -- and only fails later as an inscrutable "bundle not found" for a digest visibly +# present in the log. Asserting from inside the cluster makes that mismatch impossible to miss, +# and does it before the two model calls in Claim 3 are paid for. +if [ "$(kubectl exec -n "$NS" deploy/redis -- redis-cli EXISTS "config:bundle:$DIGEST" 2>/dev/null | tr -d '\r')" = "1" ]; then + ok "the bundle is in the cluster's Redis (verified in-cluster, not via the port-forward)" +else + ko "the bundle is NOT in the cluster's Redis — the port-forward reached a different Redis" + echo " check nothing else holds :$REDIS_PORT, then re-run" >&2 + exit 1 +fi + +# --- Claim 2: re-promotion of unchanged config uploads nothing -------------------------------- +claim 2 "re-promoting unchanged configuration uploads nothing" +repromote_log="$(mktemp)" +if ! HOME="$SANDBOX" pnpm --dir "$REPO_ROOT/harness" promote \ + --entry ship-note --project "$SANDBOX" > "$repromote_log" 2>&1; then + # Report and carry on rather than exiting: Claim 3 only needs $DIGEST from Claim 1, and an + # aborted run here would hide the A/B behind a re-promotion failure. + ko "re-promote failed" + cat "$repromote_log" >&2 +else + d2=$(grep -oE 'sha256:[0-9a-f]{64}' "$repromote_log" | head -1) + [ "$d2" = "$DIGEST" ] && ok "digest is stable" || ko "digest changed: $d2" + grep -q 'upload skipped' "$repromote_log" && + ok "upload skipped (content-addressed)" || ko "re-promotion re-uploaded the bundle" +fi + +# --- Claim 3: the A/B — same prompt, one field ------------------------------------------------ +# The bare run is the CONTROL, and on a warm cluster it is not automatically a valid one. +# +# The overlay materialises the bundle into a digest-keyed cache in the SHARED pool sandbox +# (/workspace/.sh-config//) and leaves it there for reuse. It is world-readable, it holds +# context/agents/0-CLAUDE.md and memory/, and it OUTLIVES the leaf. A later bare leaf that leases +# the same sandbox can explore the filesystem, find another run's promoted workflow and answer from +# it -- measured here: a second run of this demo had its bare arm emit "following the house rules" +# with the ticket AND the token, having been told neither. Purge the digest before the control runs, +# or the A/B silently proves nothing on every run after the first. Tracked as #216; this purge can go +# once the cache no longer outlives the leaf that created it. +claim 3 "the same prompt behaves differently only because of configRef" +POOL_SEL="${KAGENTI_SANDBOX_POOL_SELECTOR:-sh.kagenti.io/sandbox-pool=default}" +mapfile -t POOL_PODS < <(kubectl get pods -n "$NS" -l "$POOL_SEL" -o name 2>/dev/null | sed 's|pod/||') +[ "${#POOL_PODS[@]}" -gt 0 ] || { + ko "no pool sandbox pods match $POOL_SEL" + exit 1 +} +CACHE_DIR="/workspace/.sh-config/sha256-${DIGEST#sha256:}" +for p in "${POOL_PODS[@]}"; do + kubectl exec -n "$NS" "$p" -- rm -rf "$CACHE_DIR" 2>/dev/null || true +done +still=0 +for p in "${POOL_PODS[@]}"; do + kubectl exec -n "$NS" "$p" -- test -e "$CACHE_DIR" 2>/dev/null && still=$((still + 1)) +done +[ "$still" -eq 0 ] && + ok "shared config cache purged from ${#POOL_PODS[@]} pool sandbox(es), so the control is honest" || + ko "the digest is still cached in $still sandbox(es); the bare run could read it" + +SID_A="demo-bare-$$" +SID_B="demo-promoted-$$" +echo " A (bare) -> $SID_A" +bare=$(dispatch "$SID_A" "$PROMPT") +echo " B (promoted) -> $SID_B" +prom=$(dispatch "$SID_B" "$PROMPT" "$DIGEST") + +bare_text=$(jq -r '.text // ""' <<< "$bare") +prom_text=$(jq -r '.text // ""' <<< "$prom") +[ "$(jq -r '.status' <<< "$prom")" = "responded" ] || + { + ko "promoted run did not respond: $(jq -c . <<< "$prom" | head -c 300)" + exit 1 + } + +echo "" +echo " --- A, bare (no configRef) ---" +sed 's/^/ /' <<< "$bare_text" +echo " --- B, promoted ($DIGEST) ---" +sed 's/^/ /' <<< "$prom_text" +echo "" + +# Memory travelled: the id exists nowhere but the promoted memory file. +grep -q "$TICKET" <<< "$prom_text" && + ok "promoted run cites $TICKET (memory travelled)" || + ko "promoted run never cites $TICKET" +grep -q "$TICKET" <<< "$bare_text" && + ko "bare run cited $TICKET — the fact leaked from somewhere else, so the A/B proves nothing" || + ok "bare run cannot cite $TICKET" + +# The sandbox half, mechanically: the overlay must have materialised the bundle -- INCLUDING the +# skill's references/ subdirectory -- into the pool sandbox it leased. Asserted on the filesystem +# rather than in the model's words, so it holds even on a run where the model declines to read. +landed="" +for p in "${POOL_PODS[@]}"; do + if kubectl exec -n "$NS" "$p" -- test -f "$CACHE_DIR/skills/ship-note/references/release-token.md" 2>/dev/null; then + landed="$p" + break + fi +done +[ -n "$landed" ] && + ok "the bundle materialised in $landed, references/ and all (sandbox overlay ran)" || + ko "no pool sandbox holds $CACHE_DIR/skills/ship-note/references/release-token.md" + +# The sandbox half, as the model saw it: this token is only readable by a `read` executed in the +# sandbox pod, resolving a relative sibling path against the injected absolute skills root. +grep -q "$TOKEN" <<< "$prom_text" && + ok "promoted run echoes the skill's sibling token (path translation, end to end)" || + ko "no sibling token — the skill body or its references/ did not resolve in the sandbox" + +# CLAUDE.md travelled as an agents file, so this one is deterministic rather than model-dependent. +grep -qE 'RISK' <<< "$prom_text" && + ok "promoted run carries the CLAUDE.md RISK line" || + ko "no RISK line — the CLAUDE.md chain did not travel" + +# --- Claim 4: the verdict is durable and re-readable ------------------------------------------ +claim 4 "the promoted run's status is persisted and re-readable" +enc=$(jq -rn --arg s "$SID_B" '$s|@uri') +status=$(curl -s $CURL_OPTS --max-time 30 ${CURL_HDR[@]+"${CURL_HDR[@]}"} \ + "$BASE/runs/status?sessionId=$enc" || true) +[ "$(jq -r '.status // empty' <<< "$status")" = "responded" ] && + ok "/runs/status replays 'responded'" || + ko "/runs/status did not report the run: $(head -c 200 <<< "$status")" + +echo "" +echo "=== Results: $PASS passed, $FAIL failed ===" +if [ "$FAIL" -gt 0 ]; then + echo "PROMOTED-WORKFLOW DEMO FAIL" + exit 1 +fi +echo "PROMOTED-WORKFLOW DEMO PASS" diff --git a/deploy/knative/fixtures/promoted-demo/CLAUDE.md b/deploy/knative/fixtures/promoted-demo/CLAUDE.md new file mode 100644 index 0000000..42a9fc7 --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/CLAUDE.md @@ -0,0 +1,8 @@ +# Release engineering house rules + +These rules apply to every ship note, without exception. + +- Before writing a ship note you MUST load the `ship-note` skill with the read tool, and follow its + format exactly — including the release token it names. +- Every ship note ends with a `RISK:` line, rated `low`, `medium` or `high`, with a reason. +- **Never invent an incident id.** Cite the id recorded in memory, or write `no linked incident`. diff --git a/deploy/knative/fixtures/promoted-demo/commands/ship-note.md b/deploy/knative/fixtures/promoted-demo/commands/ship-note.md new file mode 100644 index 0000000..b7ba959 --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/commands/ship-note.md @@ -0,0 +1 @@ +Write the ship note for the fix described below, following the house rules exactly. diff --git a/deploy/knative/fixtures/promoted-demo/memory/MEMORY.md b/deploy/knative/fixtures/promoted-demo/memory/MEMORY.md new file mode 100644 index 0000000..5ede455 --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/memory/MEMORY.md @@ -0,0 +1 @@ +- [Auth timeout incident](auth-timeout-incident.md) — the idle reaper that closed live sessions, and the ticket tracking it diff --git a/deploy/knative/fixtures/promoted-demo/memory/auth-timeout-incident.md b/deploy/knative/fixtures/promoted-demo/memory/auth-timeout-incident.md new file mode 100644 index 0000000..432ece5 --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/memory/auth-timeout-incident.md @@ -0,0 +1,12 @@ +--- +name: auth-timeout-incident +description: The auth timeout regression, its root cause, and the ticket that tracks it +metadata: + type: project +--- + +Sessions dropped after 30 seconds of inactivity because the idle reaper measured time since the +connection _opened_ rather than time since the last byte. Tracked as **KAG-4471**. + +Fixed by resetting the reaper clock on every frame. The fix ships behind the +`auth.idle_reaper_v2` flag, so it is reversible without a redeploy. diff --git a/deploy/knative/fixtures/promoted-demo/skills/ship-note/SKILL.md b/deploy/knative/fixtures/promoted-demo/skills/ship-note/SKILL.md new file mode 100644 index 0000000..a4b4ec9 --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/skills/ship-note/SKILL.md @@ -0,0 +1,23 @@ +--- +name: ship-note +description: Use when writing a ship note, release note or changelog entry for a fix that is shipping — emits the house SHIP NOTE block. +--- + +# Ship note + +Reply with exactly this block, and nothing else: + +``` +SHIP NOTE · +WHAT: +WHY: +TICKET: +RISK: +TOKEN: +``` + +The token is **not** in this file. Read `references/release-token.md` from this skill's own +directory and copy the token verbatim into the `TOKEN:` line. + +Do not guess the token. If you cannot read that file, write `TOKEN: unavailable` — a wrong token is +worse than an absent one. diff --git a/deploy/knative/fixtures/promoted-demo/skills/ship-note/references/release-token.md b/deploy/knative/fixtures/promoted-demo/skills/ship-note/references/release-token.md new file mode 100644 index 0000000..00f174f --- /dev/null +++ b/deploy/knative/fixtures/promoted-demo/skills/ship-note/references/release-token.md @@ -0,0 +1 @@ +SHIPNOTE-7F3A-SANDBOX-OK diff --git a/deploy/knative/tests/demo-promoted-workflow.test.sh b/deploy/knative/tests/demo-promoted-workflow.test.sh new file mode 100644 index 0000000..e20f66f --- /dev/null +++ b/deploy/knative/tests/demo-promoted-workflow.test.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# deploy/knative/tests/demo-promoted-workflow.test.sh +# +# Cluster-free tests for the promoted-workflow demo. The demo itself needs a cluster and two real +# model calls, so it cannot run on every PR -- but its FIXTURE can rot silently, and a rotted +# fixture turns the demo into a green run that proves nothing. Specifically: +# +# - the A/B hinges on one unguessable string (the ticket id) living ONLY in the memory file, and +# on one token living ONLY in the skill's references/ subdirectory. If an edit moves either into +# CLAUDE.md or SKILL.md, the bare arm could produce it from the prompt alone and the demo's +# central claim quietly becomes untestable. +# - the constants the script greps for must match the fixture's contents. A typo in either makes +# every claim fail against a working cluster, which reads as a broken feature. +# +# No cluster required. Run: bash deploy/knative/tests/demo-promoted-workflow.test.sh +set -uo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIX="$DIR/fixtures/promoted-demo" +SCRIPT="$DIR/demo-promoted-workflow.sh" +fails=0 +check() { if [ "$2" = "$3" ]; then echo " ok: $1"; else + echo " FAIL: $1 (want '$3', got '$2')" + fails=$((fails + 1)) +fi; } + +echo "== fixture layout" +for f in CLAUDE.md skills/ship-note/SKILL.md skills/ship-note/references/release-token.md \ + commands/ship-note.md memory/MEMORY.md memory/auth-timeout-incident.md; do + check "$f exists" "$([ -f "$FIX/$f" ] && echo yes || echo no)" "yes" +done + +echo "== the script's constants match the fixture" +TICKET=$(grep -E '^TICKET=' "$SCRIPT" | head -1 | cut -d'"' -f2) +TOKEN=$(grep -E '^TOKEN=' "$SCRIPT" | head -1 | cut -d'"' -f2) +check "TICKET is set" "$([ -n "$TICKET" ] && echo yes || echo no)" "yes" +check "TOKEN is set" "$([ -n "$TOKEN" ] && echo yes || echo no)" "yes" +check "the ticket is in the memory file" \ + "$(grep -q "$TICKET" "$FIX/memory/auth-timeout-incident.md" && echo yes || echo no)" "yes" +check "the token is the release-token file's content" \ + "$(tr -d '[:space:]' < "$FIX/skills/ship-note/references/release-token.md")" "$TOKEN" + +echo "== the unguessable strings leak nowhere else in the fixture" +# The ticket must not be reachable from anything the harness injects into the system prompt, or the +# bare arm gets it for free and stops being a control. +for f in CLAUDE.md skills/ship-note/SKILL.md commands/ship-note.md memory/MEMORY.md; do + check "$f does not contain the ticket" \ + "$(grep -q "$TICKET" "$FIX/$f" && echo leaked || echo clean)" "clean" +done +# The token must exist ONLY in references/, so producing it requires a sandbox read. +check "SKILL.md does not contain the token" \ + "$(grep -q "$TOKEN" "$FIX/skills/ship-note/SKILL.md" && echo leaked || echo clean)" "clean" +check "CLAUDE.md does not contain the token" \ + "$(grep -q "$TOKEN" "$FIX/CLAUDE.md" && echo leaked || echo clean)" "clean" +check "the prompt the script sends does not contain the ticket" \ + "$(grep -E '^PROMPT=' "$SCRIPT" | grep -q "$TICKET" && echo leaked || echo clean)" "clean" + +echo "== the skill is loadable by pi's resolver" +# resolve.ts reads the bare frontmatter `name`; classify.ts dedupes on it. A missing or namespaced +# name silently changes the bundle path the script asserts against. +check "SKILL.md has frontmatter name: ship-note" \ + "$(awk '/^name:/{print $2; exit}' "$FIX/skills/ship-note/SKILL.md")" "ship-note" +check "SKILL.md has a description (required by the Agent Skills spec)" \ + "$(grep -cE '^description:' "$FIX/skills/ship-note/SKILL.md")" "1" +# pi puts only name+description in the system prompt and tells the model to read the body, so the +# description is what decides whether the skill is ever loaded at all. +check "the description mentions a ship note, so the task matches it" \ + "$(grep -iE '^description:.*ship note' "$FIX/skills/ship-note/SKILL.md" >/dev/null && echo yes || echo no)" "yes" + +echo "== SKILL.md instructs the sibling read the demo asserts" +check "SKILL.md names references/release-token.md" \ + "$(grep -q 'references/release-token.md' "$FIX/skills/ship-note/SKILL.md" && echo yes || echo no)" "yes" +check "SKILL.md tells the model to fail honestly rather than guess" \ + "$(grep -q 'unavailable' "$FIX/skills/ship-note/SKILL.md" && echo yes || echo no)" "yes" + +echo "== MEMORY.md links resolve (a dangling link is a preflight warning)" +while read -r target; do + [ -z "$target" ] && continue + check "MEMORY.md link '$target' exists" \ + "$([ -f "$FIX/memory/$target" ] && echo yes || echo no)" "yes" +done < <(grep -oE '\]\([^)]+\.md\)' "$FIX/memory/MEMORY.md" | sed 's/](//; s/)//') + +echo "== the script's own guards" +check "refuses a sandbox dir it did not create (marker gate)" \ + "$(grep -q 'REFUSING to touch' "$SCRIPT" && echo yes || echo no)" "yes" +check "does not default the redis forward to 6379" \ + "$(grep -qE '^REDIS_PORT=.*:-6379\}' "$SCRIPT" && echo bad || echo ok)" "ok" +check "gates on the deployed image implementing configRef" \ + "$(grep -q 'unknown digest' "$SCRIPT" && echo yes || echo no)" "yes" +check "purges the shared sandbox cache before the control run" \ + "$(grep -q 'sh-config' "$SCRIPT" && echo yes || echo no)" "yes" +check "verifies the upload landed in the cluster's redis" \ + "$(grep -q 'config:bundle:' "$SCRIPT" && echo yes || echo no)" "yes" + +echo "== --teardown is sandbox-scoped and touches no cluster" +# Same property demo-teardown-scope.test.sh pins for the remote-worker demo: a teardown must not +# delete infrastructure it did not create. This demo's teardown removes a /tmp sandbox and nothing +# else, so any kind/kubectl call on that path is a regression. kind/kubectl/docker are mocked and +# only the call log is asserted. +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +export MOCK_LOG="$TMP/calls.log" +: > "$MOCK_LOG" +mkdir -p "$TMP/bin" +for b in kind kubectl docker pnpm; do + cat > "$TMP/bin/$b" <> "\$MOCK_LOG" +exit 0 +EOF + chmod +x "$TMP/bin/$b" +done + +# A sandbox carrying the marker: teardown must remove exactly this, via plain rm, not via a cluster. +SBX="$TMP/sandbox" +mkdir -p "$SBX" +touch "$SBX/.sh-demo-sandbox" +PATH="$TMP/bin:$PATH" SH_DEMO_SANDBOX="$SBX" bash "$SCRIPT" --teardown > "$TMP/out" 2>&1 +check "teardown exits 0" "$?" "0" +check "the marked sandbox is gone" "$([ -e "$SBX" ] && echo present || echo gone)" "gone" +check "no kind calls" "$(grep -c '^kind ' "$MOCK_LOG")" "0" +check "no kubectl calls" "$(grep -c '^kubectl ' "$MOCK_LOG")" "0" + +# An unmarked directory must survive: this is the guard against a mistyped SH_DEMO_SANDBOX. +UNMARKED="$TMP/not-ours" +mkdir -p "$UNMARKED" +touch "$UNMARKED/precious.txt" +PATH="$TMP/bin:$PATH" SH_DEMO_SANDBOX="$UNMARKED" bash "$SCRIPT" --teardown > "$TMP/out2" 2>&1 +check "an unmarked directory is left alone" \ + "$([ -f "$UNMARKED/precious.txt" ] && echo kept || echo DELETED)" "kept" + +if [ "$fails" -ne 0 ]; then + echo "FAILED: $fails check(s)" + exit 1 +fi +echo "PASS" diff --git a/docs/demos/README.md b/docs/demos/README.md index 7f33b11..78ec215 100644 --- a/docs/demos/README.md +++ b/docs/demos/README.md @@ -6,10 +6,11 @@ performed live in front of an audience. ## What lives here -| Demo | Shows | Time | -| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -| [`serverless-harness-demo.md`](./serverless-harness-demo.md) | An agent that **scales to a true zero**, resumes from cold with full memory, then **fans out into a worker fleet** that appears on demand and vanishes when the queue drains | ~10 min | -| [`remote-sandbox-demo.md`](./remote-sandbox-demo.md) | A **sandbox outside the cluster** with zero inbound rules, executing a leaf's tool calls — one free-form prompt that names a different OS on each backend, and a secret planted by hand that the cluster reads back | ~10 min | +| Demo | Shows | Time | +| ------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| [`serverless-harness-demo.md`](./serverless-harness-demo.md) | An agent that **scales to a true zero**, resumes from cold with full memory, then **fans out into a worker fleet** that appears on demand and vanishes when the queue drains | ~10 min | +| [`remote-sandbox-demo.md`](./remote-sandbox-demo.md) | A **sandbox outside the cluster** with zero inbound rules, executing a leaf's tool calls — one free-form prompt that names a different OS on each backend, and a secret planted by hand that the cluster reads back | ~10 min | +| [`promoted-workflow-demo.md`](./promoted-workflow-demo.md) | A Claude Code workflow authored on your laptop — one skill, a `CLAUDE.md`, one memory file — **running unchanged in the cluster**: the same prompt twice, one field apart, and only the promoted run can cite an incident id that exists nowhere but your memory directory | ~8 min | ## Demo vs. smoke test vs. spec diff --git a/docs/demos/promoted-workflow-demo.md b/docs/demos/promoted-workflow-demo.md new file mode 100644 index 0000000..421f28d --- /dev/null +++ b/docs/demos/promoted-workflow-demo.md @@ -0,0 +1,457 @@ +# Demo: "The workflow you built on your laptop, running in the cluster" + +A ~8-minute walkthrough of **workflow promotion**: you author an agent workflow in Claude Code — a +skill, a `CLAUDE.md`, a memory file, a slash command — and run it **unchanged** in the harness by +adding one field to a dispatch. + +The task — write a ship note — is just a vehicle. The real show is **what the remote agent knows**. +You will send the same prompt twice, to the same cluster, differing only by `configRef`, and watch +one run ask what a ship note even is while the other cites an incident id that exists nowhere but +your laptop's memory directory. + +``` +laptop cluster +------ ------- +/tmp/sh-demo promote harness pod (fs-free) sandbox pod + CLAUDE.md --> canonical tar --> /tmp/sh-config// /workspace/.sh-config// + .claude/skills/ sha256 in skills/ context/ skills/ + .claude/commands/ Redis (system prompt) (what `read` can see) + .claude/projects/*/memory/ +``` + +One digest names both halves. The dispatch carries the digest and nothing else about the workflow. + +| Act | What "move my workflow to the server" normally needs | What promotion needs | +| ---------------------- | ------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | +| **1 — Author small** | A hand-written manifest listing what to ship, kept in sync by hand | **One env var.** `HOME=$SH_DEMO_SANDBOX` makes the sandbox its own user scope | +| **2 — Promote** | An image rebuild, a redeploy, a registry push | `pnpm promote` — a 12 KB tar, content-addressed; re-promotion uploads **nothing** | +| **3 — Run it** | A bespoke endpoint that knows about your skills | **One field.** `"configRef": "sha256:…"` on the existing prompt envelope | +| **4 — Know it landed** | Read the pod logs and hope | The run cites a fact only your memory holds, and a token only the sandbox can read | + +Prefer it non-interactive? `make demo-promoted-workflow` does all of this and asserts every claim. +This document is the version you drive by hand so you can explain each move. + +--- + +## Act 0: Install + +You need a **warm** harness cluster whose image contains the promotion feature (merged in +[#214](https://github.com/rossoctl/serverless-harness/pull/214)). If you do not have one: + +```bash +git clone --recurse-submodules https://github.com/rossoctl/serverless-harness.git +cd serverless-harness +cd pi-fork && npm ci && npm run build && cd .. +pnpm install + +export ANTHROPIC_API_KEY=sk-... # ...or a gateway: ANTHROPIC_BASE_URL + ANTHROPIC_AUTH_TOKEN +./deploy/knative/setup-kind.sh +``` + +> The model must be reachable **from the cluster** — both runs are real model calls. + +If your cluster is already warm but predates #214, rebuild and **force a new Revision**. The image +tag is mutable, so re-applying an unchanged spec rolls nothing and you would keep serving the old +code: + +```bash +docker build --load -t dev.local/serverless-harness:local . +kind load docker-image dev.local/serverless-harness:local --name sh-knative +kubectl -n default patch ksvc serverless-harness --type merge \ + -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"deploy.sh/build-ts\":\"$(date +%s)\"}}}}}" +kubectl wait ksvc/serverless-harness -n default --for=condition=Ready --timeout=180s +``` + +Set the convenience vars used throughout: + +```bash +export NS=default KSVC=serverless-harness +export HOSTHDR='Host: serverless-harness.default.example.com' +export BASE=http://localhost:8080 + +# The SH_* names are the ones demo-promoted-workflow.sh and the make targets read, so the +# hand-driven and scripted halves of this walkthrough stay on the same sandbox and port. +export SH_DEMO_SANDBOX=/tmp/sh-demo +export SH_DEMO_REDIS_PORT=16379 +export SH_MODEL=claude-haiku-4-5 + +# Where this repo is checked out. Every `pnpm --dir` below uses it, because from Act 1b onward +# your shell's cwd is the sandbox, not the checkout. +export HARNESS=$(pwd)/harness +``` + +### Open the two tunnels + +```bash +kubectl port-forward -n kourier-system svc/kourier 8080:80 & +kubectl port-forward -n default svc/redis 16379:6379 & +export REDIS_URL=redis://localhost:16379 +``` + +> **Why 16379 and not 6379.** A listener on 6379 is not evidence it is _this cluster's_ Redis. This +> repo's own test container (`sh-tdd-redis`) publishes `0.0.0.0:6379`; promote will happily upload +> into it, print `uploaded`, and then the harness — reading the cluster's Redis — fails with +> `config bundle not found` for the digest sitting right there in your terminal. Bind a private port +> instead. This cost the author two model calls before the failure surfaced. + +### 0a. Prove the cluster implements promotion before you trust anything else + +An unknown digest must fail **before** any model call. An image without the feature ignores the +field and answers normally — which would make Act 3's contrast look like a model mood swing rather +than a missing deployment: + +```bash +curl -s -H "$HOSTHDR" -H 'Content-Type: application/json' \ + -d '{"sessionId":"probe-1","kind":"prompt","prompt":"say hi", + "configRef":"sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}' \ + $BASE/runs | jq -c . +``` + +```json +{ "status": "failed", "reason": "error", "message": "config bundle not found: sha256:ffff…" } +``` + +> If you get `{"status":"responded", …}` instead, the cluster is serving a pre-#214 image. Stop and +> do the forced roll above. A green demo on the wrong image proves nothing. + +--- + +## Act 1: Author in a sandbox, not in your whole `~/.claude` + +> **Say this out loud, because it is the part people get wrong.** The tempting demo is "promote my +> real Claude Code setup". Don't. Measured on the author's laptop: a real `~/.claude` resolves **62 +> skills**, 56 of which travel, into an ~8.6 MB bundle with dozens of preflight warnings — nearly +> all from skills the workflow never uses. A curated sandbox ships **one** skill in **12 KB** with +> **zero** findings. That is the design's own recommendation ([spec §11]) and this act is it. + +### 1a. Build the workflow + +Four files. A skill that defines a format, a `CLAUDE.md` that sets a house rule, a memory file +holding a fact, and a slash command as the entry point: + +```bash +mkdir -p $SH_DEMO_SANDBOX/.claude/skills/ship-note/references $SH_DEMO_SANDBOX/.claude/commands +cp -R deploy/knative/fixtures/promoted-demo/skills/. $SH_DEMO_SANDBOX/.claude/skills/ +cp -R deploy/knative/fixtures/promoted-demo/commands/. $SH_DEMO_SANDBOX/.claude/commands/ +cp deploy/knative/fixtures/promoted-demo/CLAUDE.md $SH_DEMO_SANDBOX/CLAUDE.md + +# Claude Code's own memory layout: every path separator in the project path becomes '-' +MEM="$SH_DEMO_SANDBOX/.claude/projects/-$(echo "${SH_DEMO_SANDBOX#/}" | tr '/' '-')/memory" +mkdir -p "$MEM" && cp deploy/knative/fixtures/promoted-demo/memory/*.md "$MEM/" + +git -C $SH_DEMO_SANDBOX init -q + +# Claim this directory as the demo's own. demo-promoted-workflow.sh gates every destructive +# touch -- including `--teardown` -- on this marker, so without it the scripted teardown will +# (correctly) refuse to remove the sandbox you just built. +touch $SH_DEMO_SANDBOX/.sh-demo-sandbox +``` + +> **Why `git init`.** `promote` bounds its `CLAUDE.md` walk at a `.git` entry. Without one it climbs +> past your sandbox into ancestor directories and sweeps their context files — including a personal +> `~/CLAUDE.md` — into a bundle bound for a shared store. + +The memory file is the one to read aloud, because it is what makes Act 3 undeniable: + +```bash +cat "$MEM/auth-timeout-incident.md" +``` + +It records that the regression is tracked as **KAG-4471** and ships behind +`auth.idle_reaper_v2`. Nothing else in the demo knows that. + +### 1b. Watch it work locally first + +```bash +cd $SH_DEMO_SANDBOX && claude +``` + +Ask it: `Write the ship note for the auth timeout fix.` You get the house `SHIP NOTE` block with +the ticket and the risk line. This is the "works on my laptop" baseline — the thing that normally +does not survive the trip. + +> Leave this Claude Code session open. You will drive the rest of the demo _from it_: it is the +> terminal where the workflow was authored, so promoting and dispatching from here is the point. + +--- + +## Act 2: Promote + +### 2a. One command, and one env var that is the whole idea + +```bash +HOME=$SH_DEMO_SANDBOX pnpm --dir $HARNESS promote \ + --entry ship-note --project $SH_DEMO_SANDBOX +``` + +``` +project: /tmp/sh-demo +inventory: …/sandbox-inventory/ghcr.io_rossoctl_serverless-harness-sandbox_latest.json (347 binaries) + resolved 1 skills + travels 1 + dropped 0 + context 2 file(s), 1 memory file(s) + secrets no blocking findings + entry ship-note + +preflight: no findings + + bundle sha256:46ee1106… (12288 bytes, uploaded) + lockfile .claude/promoted.lock.json + +dispatch with: {"sessionId":"/","kind":"prompt","prompt":"…","configRef":"sha256:46ee1106…"} +``` + +> **`HOME=$SH_DEMO_SANDBOX` is the demo, not a shortcut.** `promote` reads _user_ scope from +> `$HOME/.claude`. Pointing `HOME` at the sandbox makes the sandbox its own user scope, so the +> bundle holds this workflow and nothing else. It is also what makes the **slash command** travel: +> `promote` reads prompts from user scope only, so a project-scope `.claude/commands/` file is +> invisible to it without this. +> +> Note `travels 1`, `preflight: no findings`, and `12288 bytes`. Those three numbers are the +> argument for sandbox-first authoring, and the last line hands you the exact dispatch envelope. + +The lockfile is committable, and it is also the most convenient place to read the digest back from: + +```bash +export DIGEST=$(jq -r .digest $SH_DEMO_SANDBOX/.claude/promoted.lock.json) +echo $DIGEST +# => sha256:46ee11062906cf70d6770a1a9c58b01429836ba4b16861b65110673904603bb4 +``` + +### 2b. Verify it landed in the cluster's Redis, not somewhere else + +Read the key back through the cluster's **own** client rather than your port-forward. A forward +pointing at the wrong Redis passes every check up to here: + +```bash +kubectl exec -n $NS deploy/redis -- redis-cli EXISTS "config:bundle:$DIGEST" +# => 1 +``` + +### 2c. Re-promotion is free + +```bash +HOME=$SH_DEMO_SANDBOX pnpm --dir $HARNESS promote --entry ship-note --project $SH_DEMO_SANDBOX | tail -3 +``` + +``` + bundle sha256:46ee1106… (12288 bytes, unchanged — upload skipped) +``` + +> Same digest, no upload. The bundle is content-addressed over a **canonical** tar — sorted paths, +> normalised mtimes and modes — so re-promoting unchanged configuration is a no-op, and the digest +> does not even depend on which directory you authored in. + +--- + +## Act 3: The same prompt, one field apart + +Both dispatches are byte-identical but for `configRef`. Same cluster, same model, same prompt. + +### 3a. Purge the shared cache first, or your control is not a control + +**Do not skip this on a warm cluster.** The overlay materialises the bundle into a digest-keyed +cache inside the **shared pool sandbox** and leaves it there for reuse. It is world-readable, it +contains `context/agents/0-CLAUDE.md` and `memory/`, and it **outlives the leaf**: + +```bash +# The leaf may have leased ANY pool sandbox, so ask the pool rather than guessing a pod. +export POOL=$(kubectl get pods -n $NS -l 'sh.kagenti.io/sandbox-pool=default' -o name | sed 's|pod/||') +for p in $POOL; do kubectl exec -n $NS "$p" -- find /workspace/.sh-config -type f 2>/dev/null; done \ + | sed 's|.*/sha256-[0-9a-f]*/||' +# context/MEMORY.md +# context/agents/0-CLAUDE.md +# memory/auth-timeout-incident.md +# skills/ship-note/SKILL.md +# skills/ship-note/references/release-token.md +``` + +> **This bit the author, so it will bite you.** On the second run of this demo the _bare_ arm +> answered `TICKET: KAG-4471 / TOKEN: SHIPNOTE-7F3A-SANDBOX-OK` and opened with "following the house +> rules" — having been told none of it. A bare leaf that leases a sandbox where a previous promoted +> leaf ran can simply _explore the filesystem_ and answer from someone else's promoted workflow. +> The A/B still looked plausible; it had just stopped proving anything. + +```bash +for p in $POOL; do + kubectl exec -n $NS "$p" -- rm -rf "/workspace/.sh-config/sha256-${DIGEST#sha256:}" +done +``` + +### 3b. Run A — bare + +```bash +curl -s -H "$HOSTHDR" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg m "$SH_MODEL" '{sessionId:"demo-bare-1", kind:"prompt", model:$m, + prompt:"Write the ship note for the auth timeout fix."}')" \ + $BASE/runs | jq -r .text +``` + +``` +The workspace appears to be empty. Could you provide me with: + +1. **Where is the code/project?** … +3. **Format for the ship note:** + - Is there an existing SHIP_NOTES, CHANGELOG, or similar file I should follow? +``` + +> It does not know what a ship note is here, has no incident to cite, and asks you. This is the +> harness's normal behaviour, unchanged — which is the back-compat guarantee, stated as output. + +### 3c. Run B — the same prompt, plus the digest + +```bash +curl -s -H "$HOSTHDR" -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg m "$SH_MODEL" --arg c "$DIGEST" '{sessionId:"demo-promoted-1", kind:"prompt", model:$m, + prompt:"Write the ship note for the auth timeout fix.", configRef:$c}')" \ + $BASE/runs | jq -r .text +``` + +``` +SHIP NOTE · auth-idle-reaper-fix +WHAT: Fixed sessions disconnecting after 30 seconds of inactivity +WHY: The idle reaper was measuring time since connection open instead of last activity +TICKET: KAG-4471 +RISK: low — fix is reversible behind auth.idle_reaper_v2 feature flag +TOKEN: SHIPNOTE-7F3A-SANDBOX-OK +``` + +> **Pause here and read the four lines back one at a time.** Each one is a different channel +> arriving, and each is separately checkable: +> +> - **The format** came from the skill. Pi puts only a skill's _name and description_ in the system +> prompt and tells the model to `read` the body when it matches — and that read runs in the +> **sandbox pod**. +> - **`TICKET: KAG-4471`** came from your memory directory. The model cannot guess it; Run A +> demonstrably did not produce it. Memory travels **read-only** +> ([ADR-0031](../adrs/0031-promoted-memory-read-only.md)), so a promoted run consumes what you +> taught it and stays replayable. +> - **`RISK:`** came from `CLAUDE.md`, injected as an agents file — the deterministic channel. +> - **`TOKEN:`** is the load-bearing one. It lives in `references/release-token.md` _inside_ the +> skill's own directory. Producing it means the bundle materialised on the **sandbox** side of the +> fs-free split **and** a relative sibling path resolved against the absolute skills root the leaf +> injects. The skill even says "if you cannot read that file, write `TOKEN: unavailable`", so a +> failed read shows up as an honest gap instead of a hallucinated token. + +### 3d. Prove the sandbox half on the filesystem, not in the model's prose + +The `TOKEN:` line is the model _telling_ you it read the file. Now check the claim directly — this +holds even on a run where the model declines to read: + +```bash +TOKEN_PATH="/workspace/.sh-config/sha256-${DIGEST#sha256:}/skills/ship-note/references/release-token.md" +for p in $POOL; do + kubectl exec -n $NS "$p" -- cat "$TOKEN_PATH" 2>/dev/null && echo " ^ in $p" && break +done +# => SHIPNOTE-7F3A-SANDBOX-OK +# ^ in sandbox-0 +``` + +> Which pod it lands in is not fixed — the leaf leases whichever pool sandbox is free, which is why +> this loops instead of naming `sandbox-0`. A hardcoded pod here fails as "broken feature" when it +> was only a wrong guess. + +> You purged this path in 3a and dispatched nothing but a digest. It is back, `references/` and all, +> because the overlay put it there under `flock` and made it read-only with `chmod -R a-w` — which +> is how [ADR-0031](../adrs/0031-promoted-memory-read-only.md)'s read-only guarantee is enforced by +> the filesystem rather than by convention. + +### 3e. Status is durable, not just a response body + +```bash +curl -s -H "$HOSTHDR" "$BASE/runs/status?sessionId=demo-promoted-1" | jq -c '{status, reason}' +# => {"status":"responded","reason":null} +``` + +> The verdict is persisted under the session id, so a re-dispatch of the same id resumes rather than +> re-pays, and a fan-out driver can collect results after the fact. This is the existing leaf +> idempotency contract; promotion did not change it. + +--- + +## What just happened + +You moved a workflow off your laptop without writing a manifest: + +1. **Authored small** — one skill, a `CLAUDE.md`, one memory file, one slash command, in a + throwaway sandbox. `HOME=$SH_DEMO_SANDBOX` made it its own user scope: **1 skill, 12 KB, zero preflight + findings**, against 56 skills and ~8.6 MB for a real `~/.claude` (Act 1, 2a). +2. **Promoted with one command** — content-addressed over a canonical tar, so the second promotion + uploaded nothing and the digest was identical (Act 2a, 2c). +3. **Dispatched with one field** — `configRef` on the existing prompt envelope. No new endpoint, no + redeploy, no image rebuild (Act 3c). +4. **Proved all three channels arrived** — the skill's format, the `CLAUDE.md` rule, and an + incident id that exists only in your memory directory, with a bare run standing next to it as a + control you made honest by purging the shared cache first (Act 3a, 3b, 3c). +5. **Proved the sandbox half twice** — once in the model's words (a token readable only by a `read` + executed in the separate sandbox pod, through a path the leaf translated) and once on the + filesystem, which does not depend on the model cooperating (Act 3c, 3d). +6. **Kept the old contract** — the bare run behaved exactly as the harness always did, and + `/runs/status` replayed the verdict (Act 3b, 3e). + +To replay all of it non-interactively with every claim asserted: + +```bash +make demo-promoted-workflow +``` + +--- + +## Cleanup + +```bash +make demo-promoted-workflow-teardown # removes the sandbox (needs the Act 1a marker) +pkill -f 'kubectl port-forward -n kourier-system svc/kourier' +pkill -f 'kubectl port-forward -n default svc/redis' +``` + +The bundle stays in Redis under a 30-day TTL. It is immutable and content-addressed, so leaving it +costs 12 KB and makes the next run of this demo skip the upload. To drop it: + +```bash +kubectl exec -n $NS deploy/redis -- redis-cli DEL "config:bundle:$DIGEST" +``` + +--- + +## Notes and limits + +- **The `TOKEN:` line is the one model-dependent claim.** The format and the token both require the + model to choose to `read` the skill body. `CLAUDE.md` says it MUST, and the deterministic channel + carries that instruction, so it is reliable in practice — but it is not a mechanical guarantee the + way `TICKET:` is. If it ever comes back `unavailable`, that is the skill being honest, not the + overlay silently failing. +- **The shared config cache is visible to other leaves, and that is worth saying out loud.** The + digest-keyed directory in the pool sandbox is what makes reuse cheap, and the overlay makes it + read-only — but read-only is not invisible. Any leaf leasing that sandbox can read another + workflow's promoted `CLAUDE.md` and `memory/`, with or without a `configRef` of its own. Act 3a + works around it for the demo's sake. Tracked as + [#216](https://github.com/rossoctl/serverless-harness/issues/216): the narrow point is that the + cache outlives the leaf that made it, so a `configRef`-less leaf can answer from it — which makes + spec §2 goal 6 ("absent a promoted bundle, harness behavior is unchanged") true in the harness + process but not observably true. Cross-leaf reading itself is an accepted non-goal (P2 §9, one + trust domain; Kata isolation is P3/#48), and ADR-0031 speaks to write-protection rather than to + visibility or lifetime. Once the cache no longer outlives its leaf, Act 3a's purge can go. +- **This demo does not show MCP servers or subagents.** Both are explicitly out of scope for + promotion (spec §2, §9). Do not let a room infer that a promoted workflow carries its MCP config. +- **Memory is read-only in the promoted run.** The remote agent cannot write back what it learns; + discoveries surface in the leaf result instead. That is deliberate + ([ADR-0031](../adrs/0031-promoted-memory-read-only.md)) and it is what keeps leaf replay + reproducible. +- **`promote` reads slash commands from user scope only.** A project-scope `.claude/commands/` file + does not travel unless `HOME` points at that project — which the sandbox pattern does anyway, but + it will surprise someone promoting a real repo. +- **Preflight blocks on facts and warns on heuristics.** The secret scan refuses the upload only on + structural credential formats (AWS key ids, PEM blocks, GitHub/Slack/OpenAI tokens); prose-shaped + matches warn. Do not promise that promotion cannot leak a pasted password — it can, and it warns. + A curated sandbox like this one yields zero findings, which is the state in which blocking + preflight would become safe to restore (spec §6 D10, §11). +- **`--dry-run` shows you the bundle without uploading it**, which is the safe way to inspect what a + real `~/.claude` would ship before you ship it. + +[spec §11]: ../specs/2026-09-02-claude-code-workflow-promotion-design.md + +Reference: [`../../deploy/knative/demo-promoted-workflow.sh`](../../deploy/knative/demo-promoted-workflow.sh), +[ADR-0030](../adrs/0030-claude-code-workflow-promotion.md), +[ADR-0031](../adrs/0031-promoted-memory-read-only.md). diff --git a/docs/specs/2026-09-02-claude-code-workflow-promotion-design.md b/docs/specs/2026-09-02-claude-code-workflow-promotion-design.md index 41aed9c..8293378 100644 --- a/docs/specs/2026-09-02-claude-code-workflow-promotion-design.md +++ b/docs/specs/2026-09-02-claude-code-workflow-promotion-design.md @@ -1,6 +1,6 @@ # Claude Code workflow promotion: design -**Date:** 2026-09-02 · **Status:** Implemented (cold-start measurement owed) · **ADRs:** +**Date:** 2026-09-02 · **Status:** Implemented · **ADRs:** [ADR-0030](../adrs/0030-claude-code-workflow-promotion.md), [ADR-0031](../adrs/0031-promoted-memory-read-only.md) · **Builds on:** M1 (Redis session backend), M2/M3 (sandbox client + persistent channel), P1 (fs-free harness), P2 (shared @@ -467,19 +467,38 @@ plausible thing to erode it, so the cost belongs in the evidence trail rather th assertion. The end-to-end, in-cluster comparison (baseline vs. `configRef` set, against a scaled-to-zero Revision) could not be taken during this implementation: the deployed image predates this branch, and taking it needs a full image build, `kind load`, and a forced new -Revision. It remains owed; reproduction, once such a cluster is available: +Revision. It has since been taken (see **In-cluster result**, below). + +The snippet originally recorded here forced `--replicas=0` and slept 5 s. Do not use it: it does +not produce a cold start (see the second false start below), and its selector named `harness` +rather than `serverless-harness`. What was actually run: ```bash -# baseline: no configRef +# Wait for Knative's OWN scale-to-zero -- do not force --replicas=0, the KPA re-scales. +wait_zero() { + local waited=0 + while [ "$waited" -lt 240 ]; do + [ "$(kubectl -n "$NS" get pods -l serving.knative.dev/service="$KSVC" \ + --no-headers 2>/dev/null | grep -c Running)" = "0" ] && return 0 + sleep 5; waited=$((waited + 5)) + done + return 1 +} + +# Alternate the arms so autoscaler and network drift hit both. A fresh output file per sample, +# and a non-200 is a hard failure: reusing one path let a dead tunnel report stale successes. for i in 1 2 3 4 5; do - kubectl -n "$NS" scale deployment -l serving.knative.dev/service=harness --replicas=0 2>/dev/null || true - sleep 5 - curl -s -o /dev/null -w '%{time_total}\n' -X POST "$KSVC_URL/runs" \ - -H 'content-type: application/json' \ - -d '{"sessionId":"cold-base/'"$i"'","kind":"prompt","prompt":"Say PONG"}' + for ref in '' '"configRef":"",'; do + wait_zero || { echo "not cold"; continue; } + out=$(mktemp) + r=$(curl -s -o "$out" -w '%{time_total} %{http_code}' --max-time 300 \ + -H "$HOSTHDR" -H 'content-type: application/json' \ + -d '{"sessionId":"cold-'"$i"'","kind":"prompt",'"$ref"'"prompt":"Say PONG and nothing else."}' \ + "$BASE/runs") + [ "${r##* }" = "200" ] && echo "${r%% *}" || echo "FAILED http=${r##* }" + rm -f "$out" + done done - -# with a promoted bundle: same, adding "configRef":"" ``` What IS measured, locally, is the cost promote's cold path adds beyond the existing inline @@ -493,6 +512,33 @@ APFS rather than in-cluster (in a pod, Redis is a network hop and the unpack tar emptyDir on node disk); excludes container start, which dominates real cold start; excludes loader init. +**In-cluster result (2026-09-03).** Taken on kind against a Revision built from `main` with +this feature in it — the image blocker above was cleared by building the promoted-workflow demo +([`../demos/promoted-workflow-demo.md`](../demos/promoted-workflow-demo.md)). N=5 per arm, +alternating baseline/promoted, each sample preceded by a wait for Knative's own scale-to-zero, one +real model call per sample (`claude-haiku-4-5`), promoted arm carrying a 12 KB bundle: + +| arm | median | min | max | +| -------------------------- | ------ | ------ | ------- | +| baseline (no `configRef`) | 6.38 s | 6.18 s | 7.43 s | +| promoted (`configRef` set) | 7.61 s | 6.23 s | 15.31 s | + +**Median delta +1234 ms, which this measurement cannot resolve.** Within-arm spread reaches +9082 ms, so the delta sits well below the noise floor: end-to-end cold start here is +dominated by container start and one model call, both of which swamp the ~114 ms of bundle fetch, +verify and unpack measured in isolation above. The honest reading is **no cold-start regression +observable at this sample size**, not "promotion costs +1234 ms". Anyone wanting a resolvable +number should measure `getBundle`+`unpackBundle` in-pod directly rather than through a cold dispatch, +or raise N by an order of magnitude. + +Two false starts are recorded because both produced confident, wrong numbers. Reusing a single curl +output path let a dead port-forward report `0.0005 s responded` for ten straight samples — curl never +wrote the file and `jq` read the previous body; the driver now uses a fresh file per sample and fails +on a non-200. And forcing `kubectl scale --replicas=0` fights the KPA, which re-scales after each +served request: the first force after a served request never reached zero inside 60 s while the next +did, so every baseline sample was skipped and every promoted sample ran — a systematically one-armed +"comparison" that still printed five tidy promoted timings. + **Done means:** 1. A bundle promoted from a real `~/.claude` runs a leaf that invokes a promoted skill and @@ -504,7 +550,8 @@ loader init. 5. Re-promoting unchanged configuration uploads nothing. 6. The harness's own `CLAUDE.md` is provably absent from a promoted session. 7. Added cold-path cost (bundle fetch, verify, unpack) measured locally — 113.7 ms median; the - end-to-end, in-cluster cold-start delta remains owed (see §8 Measurement). + end-to-end, in-cluster cold-start comparison taken on kind — median delta +1234 ms, below this + measurement's noise floor, i.e. no observable regression (see §8 Measurement). 8. The lockfile is committed and diffs legibly between promotions. Continuing the red-team precedent from the fs-free spec: **a grep assertion that no bundle diff --git a/harness/test/promote-live-smoke.test.ts b/harness/test/promote-live-smoke.test.ts index e8f9607..e80e676 100644 --- a/harness/test/promote-live-smoke.test.ts +++ b/harness/test/promote-live-smoke.test.ts @@ -24,6 +24,14 @@ afterAll(async () => { for (const c of clients) await c.quit(); }); +// Session ids here must NOT use the `/` shape seen in leaf-smoke.sh and the leaf +// fixtures. Those go through an item envelope, where `leafSessionId` derives a sanitized id; these +// call `runLeaf` with `kind: 'prompt'`, whose sessionId is used verbatim as the session key and is +// validated against `[alnum][alnum._-]*[alnum]`. With a slash, the first test failed in 597 ms with +// "Session id must be non-empty, contain only alphanumeric characters..." -- before any model call, +// so it never reached the assertion it exists to make. The second test passed only incidentally: +// an unknown digest fails before session-id validation is reached, so its `toContain('not found')` +// held while its id was equally invalid. describe('promoted workflow, end to end', () => { it.runIf(LIVE)( 'runs a promoted skill that reads its own sibling file in the sandbox', @@ -44,7 +52,7 @@ describe('promoted workflow, end to end', () => { await putBundle(client as unknown as BundleRedisLike, built.digest, built.tar); const result = await runLeaf({ - sessionId: `promote-smoke/${Date.now()}`, + sessionId: `promote-smoke-${Date.now()}`, item: { item_id: 'i1', file: 'f', pattern: 'p' }, kind: 'prompt', prompt: 'Reply with exactly the secret word and nothing else.', @@ -63,7 +71,7 @@ describe('promoted workflow, end to end', () => { it.runIf(LIVE)('fails loudly on an unknown digest instead of running unconfigured', async () => { const result = await runLeaf({ - sessionId: `promote-smoke-missing/${Date.now()}`, + sessionId: `promote-smoke-missing-${Date.now()}`, item: { item_id: 'i1', file: 'f', pattern: 'p' }, kind: 'prompt', prompt: 'anything',