diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9618c97..c111260 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,20 @@ jobs: echo "::error::the always-run anchor step recorded '$GATE_HEAD', not the observed head" exit 1 fi + recipe-openhands: + name: recipe (openhands) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + # openhands-sdk requires >=3.12; the certifier itself is stdlib-only, so + # its behavioural e2e already runs on the whole gates matrix. + python-version: "3.12" + - name: Install recipe dependencies + run: python -m pip install --upgrade pip pyyaml pytest jsonschema openhands-sdk==1.37.1 openhands-tools==1.37.1 + - name: OpenHands schema-drift alarm + run: python -B -m pytest -q -p no:cacheprovider scripts/test_openhands_sdk_drift.py scripts/test_openhands_recipe.py action-dogfood: name: action (dogfood on flagship example) diff --git a/docs/gap-reports/scoreboard.md b/docs/gap-reports/scoreboard.md index 7c4995d..1d0463a 100644 --- a/docs/gap-reports/scoreboard.md +++ b/docs/gap-reports/scoreboard.md @@ -57,7 +57,16 @@ for exactly the machinery our signals miss. Harnesses whose run state lives fundamentally off-repo (OpenHands and SWE-agent trajectories, platform-hosted runs) are out of scope: there is no on-disk run record for a repo-native inspector to read — which is its own -answer to the question this scoreboard asks. +answer to the question this scoreboard asks.† + +> † **Correction (2026-07-25).** For OpenHands this is now only half true. The +> V1 SDK (`openhands-sdk` 1.37.1) persists `base_state.json` + an `events/` +> trajectory whenever `persistence_dir=` is set — enough for an external +> certifier to read a run's terminal signal, its iteration cap, and its full +> event log. It is still not a *repo-native* contract (the record lives outside +> the repo by default and carries no spec, plan, or ledger), so the row stays +> out of the scoreboard; but the gap is addressable from outside, which is what +> [`examples/openhands-certify/`](../../examples/openhands-certify/) does. ## The scoreboard @@ -283,8 +292,10 @@ FCR/RP — is a missing **layer**, one every harness on this board could emit at its finish line. The port is small: four `loop.emit` calls (`open_contract`, `append_iteration`, `append_receipt`, `terminate`), worked end-to-end for a real engine in -[`docs/integrations/langgraph.md`](../integrations/langgraph.md) and -[`docs/integrations/temporal.md`](../integrations/temporal.md). +[`docs/integrations/langgraph.md`](../integrations/langgraph.md), +[`docs/integrations/temporal.md`](../integrations/temporal.md), and — for a +harness with no seam to hook at all — +[`docs/integrations/openhands.md`](../integrations/openhands.md). ## Contribute a row diff --git a/docs/integrations/openhands.md b/docs/integrations/openhands.md new file mode 100644 index 0000000..8037ebe --- /dev/null +++ b/docs/integrations/openhands.md @@ -0,0 +1,118 @@ +# OpenHands — certify the run after it ends + +OpenHands owns the EXECUTE tier: the autonomous coding runtime that plans, edits, +runs commands, and decides for itself when it is done. What it cannot do is +independently check *its own* completion claim — the agent both does the work and +declares `FINISHED`. Loop Engineer adds the tier *above* it: a contract-and-proof +layer that turns "the conversation finished" into evidence-backed proof-of-done. +It never replaces OpenHands; it certifies what OpenHands ran. + +## The pattern + +Unlike LangGraph (`END` edge) or Temporal (certify activity), OpenHands has **no +seam to insert a certify node into** — the run ends when the agent sets its own +execution status. So the recipe is a **post-run certifier** over the record the SDK +already persisted: + +``` +//base_state.json # execution_status, max_iterations, stats +//events/event-00000-.json ← the trajectory + events/event-00001-.json +``` + +The conversation id segment is the UUID **hex** (32 chars, no hyphens); a +conversation only persists when you pass `persistence_dir=`. The certifier takes a +conversation dir directly, so that composition rule is documentation, not code. + +```python +from loop import emit +from loop.integrations import EngineOutcome, to_terminal_state + +state = json.loads((conv_dir / "base_state.json").read_text(encoding="utf-8")) +events = sorted((conv_dir / "events").glob("event-*.json"), key=event_index) + +gate = holdout_gate.decide(visible, withheld) # visible green + withheld green? +ac = anticheat_scan.scan(diff_text=git_diff, trajectory=[str(p) for p in events]) +terminal = to_terminal_state( + outcome=to_engine_outcome(state, events, artifacts=[...]), + gate_verdict=gate, anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, +) +emit.terminate(ws, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1) +``` + +The certifier **imports no `openhands` package** — the record is plain JSON, so it +runs on Python 3.10 even though the SDK requires 3.12, and the gate needs no LLM +key. The event log doubles as the trajectory fed to `anticheat_scan.scan`: the +"the runtime ran the tests but the agent read the answer key" case OpenHands +cannot catch about itself, because the answer key is not part of its contract. + +## OpenHands signal → typed terminal state + +| `base_state.json` signal | `EngineOutcome` field | Typed terminal state | +|---|---|---| +| `execution_status: "finished"`, gate green + anticheat clean | `reached_end=True` | `Succeeded` | +| `execution_status: "finished"`, visible green / withheld red | `reached_end=True` | `FailedUnverifiable` (`false_completion: true`) | +| `execution_status: "stuck"` (stuck detector) | `budget_exhausted=True` | `FailedBudget` | +| `execution_status: "error"` + `ConversationErrorEvent.code == "MaxIterationsReached"` | `budget_exhausted=True` | `FailedBudget` | +| `execution_status: "error"`, any other code | `external_error=": "` | `FailedBlocked` | +| `execution_status: "paused"` (`conversation.pause()`) | `human_abort=True` | `AbortedByHuman` | +| `idle` / `running` / `waiting_for_confirmation` (read mid-flight or abandoned) | `reached_end=False` | `FailedUnverifiable` | +| trajectory touched an answer-key path (anticheat HIGH) | — | `FailedUnverifiable` | +| the diff edits a gate script (anticheat CRITICAL) | — | `FailedSafety` | + +**The precedence trap.** A max-iteration stop arrives *as* +`execution_status == "error"` with a `ConversationErrorEvent` whose `code` is +`MaxIterationsReached`. Since `to_terminal_state` ranks blocked above budget, +setting **both** `external_error` and `budget_exhausted` reports `FailedBlocked` +and silently loses the budget signal. Inspect the error code first and set +**exactly one**. `code` is a free-form `str`, so anything unrecognized falls +through to `FailedBlocked` — which fails safe: an unclassified error can never +become `Succeeded`. + +## Zero-install mode + +The `loop.integrations` module is convenience, not a requirement — the whole +projection is the SAME ~15 lines the LangGraph and Temporal recipes paste +(the adapter is engine-neutral): + +```python +def to_terminal(gate, anticheat, criteria_met, evidence, + *, human_abort=False, blocked=None, over_budget=False): + fc = gate.get("false_completion") is True + if anticheat.get("downgrade_to") == "FailedSafety": state = "FailedSafety" + elif human_abort: state = "AbortedByHuman" + elif blocked: state = "FailedBlocked" + elif over_budget: state = "FailedBudget" + elif any(v is None for v in criteria_met.values()): state = "FailedSpecGap" + elif (not gate or not anticheat or anticheat.get("downgrade_to") + or gate.get("verdict") != "Succeeded" or fc + or not any(criteria_met.values()) or not evidence): state = "FailedUnverifiable" + else: state = "Succeeded" + return {"schema": "loop-engineer/terminal@1", "state": state, + "criteria_met": {k: v is True for k, v in criteria_met.items()}, + "evidence": list(evidence), "false_completion": fc} +``` + +## Gate it in CI + +```yaml +- run: pip install loop-engineer +- run: python certify_run.py run/ --conversation "$CONV_DIR" --agent-workspace "$WS" +- run: loop doctor run/ # -> {"ok": true}: the contract is structurally honest +- run: loop metrics run/ # -> false_completion_rate + evidence-backed scorecard +``` + +`loop metrics` scores the run from its on-disk evidence — not the agent's narration. + +Verified against `openhands-sdk` 1.37.1 (2026-07-25), MIT +([`OpenHands/software-agent-sdk`](https://github.com/OpenHands/software-agent-sdk), +"Copyright (c) 2026 OpenHands contributors" — PyPI carries no license metadata). +The persistence layout, the `ConversationExecutionStatus` members, and the +`MaxIterationsReached` literal are pinned live by +[`scripts/test_openhands_sdk_drift.py`](../../scripts/test_openhands_sdk_drift.py). + +Full runnable example (six committed fixture conversations + the false-completion +demo): [`examples/openhands-certify/`](../../examples/openhands-certify/). diff --git a/docs/superpowers/specs/2026-06-30-st3-integration-adapters.md b/docs/superpowers/specs/2026-06-30-st3-integration-adapters.md index c12ad36..7050852 100644 --- a/docs/superpowers/specs/2026-06-30-st3-integration-adapters.md +++ b/docs/superpowers/specs/2026-06-30-st3-integration-adapters.md @@ -262,6 +262,15 @@ truth. ### 5.3 OpenHands run → FCR gate *(alternate)* +> **Superseded 2026-07-25 by the shipped recipe.** OpenHands restructured into +> "V1": the runtime moved to `OpenHands/software-agent-sdk`, and none of the +> `openhands.run()` / `result.iterations` / `AgentStuckError` surface sketched +> below exists. The shape below (post-run hook, trajectory as anticheat input, +> stuck/max-iteration → `FailedBudget`) survived the rewrite intact; the API did +> not. Author against [`docs/integrations/openhands.md`](../../integrations/openhands.md) +> and [`examples/openhands-certify/`](../../../examples/openhands-certify/), not +> against this snippet. + **Composes:** the EXECUTE tier (autonomous coding runtime). OpenHands writes, runs, and tests code in a sandbox — incidental verification, but "done" is still the agent stopping. Loop Engineer wraps the run's exit in the false-completion gate. diff --git a/examples/openhands-certify/README.md b/examples/openhands-certify/README.md new file mode 100644 index 0000000..4cc9293 --- /dev/null +++ b/examples/openhands-certify/README.md @@ -0,0 +1,85 @@ +# OpenHands recipe — certify the run after it ends + +A runnable post-run certifier for [OpenHands](https://github.com/OpenHands/software-agent-sdk). +OpenHands keeps its own runtime; Loop Engineer adds the contract/proof tier above +it — evidence-backed state the `loop` CLI can independently validate and score. + +## Why post-run + +LangGraph has an `END` edge and Temporal has a certify activity. OpenHands has +neither: a run ends when the agent itself sets `execution_status = FINISHED`. +The seam that needs **zero engine changes** is therefore the record the SDK +already wrote when you pass `persistence_dir=`: + +``` +//base_state.json +//events/event-00000-.json +``` + +`certify_run.py` reads that with nothing but `json` — it imports no `openhands` +package, so it runs on Python 3.10 while the SDK requires 3.12, and it needs no +LLM key. + +## What it shows + +```bash +python certify_run.py demo-run/ \ + --conversation fixtures/conversations/finished \ + --agent-workspace fixtures/workspaces/green +loop doctor demo-run/ # -> {"ok": true, ...} +loop metrics demo-run/ # -> clean scorecard +``` + +The certifier runs the same **visible + withheld** split the loop optimized +against through the real `holdout_gate.decide`, sweeps the event log through +`anticheat_scan.scan`, projects the OpenHands terminal through +`to_terminal_state`, and records it via `loop.emit`. It writes two evidence +artifacts a scorecard can join: the verbatim gate verdict +(`holdout-verdict.json`) and a verify bundle (`verify-T1.json`). + +On a real pass the terminal is `Succeeded` with evidence, and `loop metrics` +scores the run clean: `false_completion_rate 0.0`, `evidence_backed: true`, the +two FCR methods agree. + +### The false-completion demo + +```bash +python certify_run.py sabotaged-run/ \ + --conversation fixtures/conversations/finished \ + --agent-workspace fixtures/workspaces/stale +``` + +Same conversation record — OpenHands still reports `finished` — but the work +product passes the **visible** check (the file exists) and fails the **withheld** +one (the content is wrong). That is the measurable false-completion event: the +terminal becomes `FailedUnverifiable` with `false_completion: true`, **never** +`Succeeded`. The dishonest completion is recorded, not laundered. + +## Fixtures + +`fixtures/conversations/` holds six conversation dirs captured from +`openhands-sdk` 1.37.1 (trimmed: the agent/LLM block is reduced, the system-prompt +event dropped; every field the certifier reads is verbatim). + +| Fixture | `execution_status` | Terminal (with a green workspace) | +|---|---|---| +| `finished` | `finished` | `Succeeded` — or `FailedUnverifiable` with the `stale` workspace | +| `max-iterations` | `error` + `MaxIterationsReached` | `FailedBudget` | +| `stuck` | `stuck` | `FailedBudget` | +| `blocked` | `error` + `LLMAuthenticationError` | `FailedBlocked` | +| `paused` | `paused` | `AbortedByHuman` | +| `running` | `running` | `FailedUnverifiable` | + +Every non-happy row is certified against a **green** workspace on purpose: a +passing check never overrides the engine's own terminal signal. + +Because the fixtures are committed, `scripts/test_openhands_recipe.py` is +deterministic and credential-free and runs in the default gates matrix. +`scripts/test_openhands_sdk_drift.py` pins those fixtures against the *installed* +SDK and is the live schema-drift alarm (its own CI job, python 3.12). + +## The general pattern + +The complement framing, the full signal table, the precedence trap, and the +copy-paste (zero-install) projection live in +[`docs/integrations/openhands.md`](../../docs/integrations/openhands.md). diff --git a/examples/openhands-certify/certify_run.py b/examples/openhands-certify/certify_run.py new file mode 100644 index 0000000..08e6005 --- /dev/null +++ b/examples/openhands-certify/certify_run.py @@ -0,0 +1,176 @@ +"""Certify a finished OpenHands run from its persisted Conversation record. + +OpenHands has no "certify node" seam — a run ends when the agent sets its own +execution status. So the recipe is a POST-RUN CERTIFIER: it reads the +conversation directory the SDK already wrote (``base_state.json`` + +``events/event-*.json``), runs the same visible + withheld-holdout split the +loop optimized against through the real gate, projects the engine terminal +through ``loop.integrations``, and records it via ``loop.emit`` — which refuses +a dishonest ``Succeeded`` before anything hits disk. + + python certify_run.py --conversation --agent-workspace + +It imports no ``openhands`` package: the layout is documented and the record is +plain JSON, so the certifier stays stdlib-only on Python 3.10 while the SDK +itself requires 3.12. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from loop import emit +from loop._resources import tools_dir +from loop.integrations import EngineOutcome, to_terminal_state + +sys.path.insert(0, str(tools_dir())) +import anticheat_scan # noqa: E402 +import holdout_gate # noqa: E402 + +# openhands-sdk 1.37.1: openhands/sdk/conversation/persistence_const.py +BASE_STATE = "base_state.json" +EVENTS_DIR = "events" +EVENT_GLOB = "event-*.json" +ERROR_EVENT_KIND = "ConversationErrorEvent" +MAX_ITERATIONS_CODE = "MaxIterationsReached" + +EXPECTED = "hello from openhands\n" + + +def _event_order(path: Path) -> tuple[int, str]: + """``event-{idx:05d}-{uuid}.json`` — past 99999 the index outgrows its zero + padding and lexical order stops agreeing with write order.""" + parts = path.name.split("-", 2) + try: + return (int(parts[1]), path.name) + except (IndexError, ValueError): + return (-1, path.name) + + +def read_conversation(conv_dir: str | Path) -> dict: + """Pure-stdlib reader over the SDK's on-disk layout, tolerant of unknown + fields — ``base_state.json`` is a dump of a fast-moving pydantic model, so + only ``execution_status`` and the event log are treated as contract.""" + conv_dir = Path(conv_dir) + state = json.loads((conv_dir / BASE_STATE).read_text(encoding="utf-8")) + events = sorted((conv_dir / EVENTS_DIR).glob(EVENT_GLOB), key=_event_order) + return {"state": state, "event_paths": [str(p) for p in events]} + + +def last_error(event_paths: list[str]) -> tuple[str, str]: + """The last ``ConversationErrorEvent``'s ``(code, detail)``; ``("", "")`` if + the run recorded none.""" + for path in reversed(event_paths): + event = json.loads(Path(path).read_text(encoding="utf-8")) + if event.get("kind") == ERROR_EVENT_KIND: + return str(event.get("code", "")), str(event.get("detail", "")) + return "", "" + + +def to_engine_outcome(record: dict, artifacts: list[str]) -> EngineOutcome: + """Project ``execution_status`` (+ the error code) onto EngineOutcome. + + Exactly ONE of ``external_error`` / ``budget_exhausted`` is ever set: a + max-iteration stop arrives as ``execution_status == "error"``, and since + blocked outranks budget in ``to_terminal_state``, setting both would report + ``FailedBlocked`` and silently lose the budget signal. + """ + status = str(record["state"].get("execution_status", "")).lower() + code, detail = last_error(record["event_paths"]) + + if status == "stuck": + return EngineOutcome(reached_end=False, budget_exhausted=True, artifacts=artifacts) + if status == "error": + if code == MAX_ITERATIONS_CODE: + return EngineOutcome(reached_end=False, budget_exhausted=True, artifacts=artifacts) + # ConversationErrorEvent.code is a free-form str and the event may be + # absent entirely; an empty external_error would fall through to the + # gate, so an unclassified error still has to name itself. + blocked = ": ".join(part for part in (code, detail) if part) or "unclassified engine error" + return EngineOutcome(reached_end=False, external_error=blocked, artifacts=artifacts) + if status == "paused": + return EngineOutcome(reached_end=False, human_abort=True, artifacts=artifacts) + return EngineOutcome(reached_end=(status == "finished"), artifacts=artifacts) + + +def certify(out_dir: Path, conv_dir: Path, agent_workspace: Path) -> dict: + record = read_conversation(conv_dir) + artifact = agent_workspace / "artifact.txt" + + # 1. The gate: visible = what the run optimized against; withheld = the rest. + visible = [{"id": "artifact-exists", "passed": artifact.is_file()}] + withheld = [{ + "id": "artifact-content", + "passed": artifact.is_file() and artifact.read_text(encoding="utf-8") == EXPECTED, + }] + gate = holdout_gate.decide(visible, withheld) + # The event log IS the trajectory — the "the runtime ran the tests but the + # agent read the answer key" case OpenHands cannot catch about itself. + ac = anticheat_scan.scan(diff_text="", trajectory=record["event_paths"]) + + # 2. Evidence artifacts: the gate verdict + a verify bundle metrics can join. + art_dir = out_dir / ".loop" / "artifacts" + art_dir.mkdir(parents=True, exist_ok=True) + (art_dir / "holdout-verdict.json").write_text(json.dumps(gate, indent=2) + "\n", encoding="utf-8") + bundle = { + "task": "T1", + "verify": "post-run certifier — holdout_gate.decide over visible+withheld", + "outcome": "PASS" if gate["verdict"] == "Succeeded" else "FAIL", + "iteration_id": 1, + "criteria": {"1": gate["verdict"] == "Succeeded"}, + } + (art_dir / "verify-T1.json").write_text(json.dumps(bundle, indent=2) + "\n", encoding="utf-8") + + # 3. Project the OpenHands terminal into a typed state; write via emit only. + terminal = to_terminal_state( + outcome=to_engine_outcome( + record, + [".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + ), + gate_verdict=gate, + anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + passed = terminal["state"] == "Succeeded" + status = str(record["state"].get("execution_status", "")).lower() + emit.append_iteration( + out_dir, iteration_id=1, outcome="task_passed" if passed else "task_failed", + task_id="T1", + actions=[ + f"read conversation record ({len(record['event_paths'])} events)", + "ran holdout_gate.decide + anticheat_scan.scan", + ], + verify_cmd="holdout_gate.decide(visible, withheld)", verify_outcome=gate["verdict"], + notes=f"execution_status: {status}; verify bundle: verify-T1.json; " + f"gate verdict: holdout-verdict.json", + ) + emit.append_receipt(out_dir, iteration_id=1, role="orchestrate", model="deterministic-demo", outcome="ok") + emit.terminate( + out_dir, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return terminal + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("out_dir", help="fresh directory for the emitted loop contract") + parser.add_argument("--conversation", required=True, + help="the SDK conversation dir (holds base_state.json + events/)") + parser.add_argument("--agent-workspace", required=True, + help="the workspace the run produced (base_state.json records it " + "as workspace.working_dir)") + args = parser.parse_args(argv) + + emit.open_contract(args.out_dir) + terminal = certify(Path(args.out_dir), Path(args.conversation), Path(args.agent_workspace)) + print(f"terminal: {terminal['state']} — validate: python3 -m loop doctor {args.out_dir}") + return 0 if terminal["state"] == "Succeeded" else 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/examples/openhands-certify/fixtures/conversations/blocked/base_state.json b/examples/openhands-certify/fixtures/conversations/blocked/base_state.json new file mode 100644 index 0000000..853dbd1 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/blocked/base_state.json @@ -0,0 +1,54 @@ +{ + "id": "00000000-0000-4000-8000-000000000004", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": [ + "FinishTool", + "ThinkTool" + ], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000004", + "max_iterations": 500, + "stuck_detection": true, + "execution_status": "error", + "confirmation_policy": { + "kind": "NeverConfirm" + }, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": { + "secret_sources": {} + }, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/blocked/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/blocked/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..9b47fef --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/blocked/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,20 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [ + { + "cache_prompt": false, + "type": "text", + "text": "write artifact.txt" + } + ], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/blocked/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab02.json b/examples/openhands-certify/fixtures/conversations/blocked/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab02.json new file mode 100644 index 0000000..66e8844 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/blocked/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab02.json @@ -0,0 +1,9 @@ +{ + "id": "b1c2d3e4-0000-4000-8000-00000000ab02", + "timestamp": "2026-07-25T12:11:04.510221", + "source": "environment", + "parent_id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "code": "LLMAuthenticationError", + "detail": "the configured provider rejected the credentials", + "kind": "ConversationErrorEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/finished/base_state.json b/examples/openhands-certify/fixtures/conversations/finished/base_state.json new file mode 100644 index 0000000..6d1fe50 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/finished/base_state.json @@ -0,0 +1,47 @@ +{ + "id": "00000000-0000-4000-8000-000000000001", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": ["FinishTool", "ThinkTool"], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000001", + "max_iterations": 500, + "stuck_detection": true, + "execution_status": "finished", + "confirmation_policy": {"kind": "NeverConfirm"}, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": {"secret_sources": {}}, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/finished/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/finished/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..4e17137 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/finished/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,14 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [{"cache_prompt": false, "type": "text", "text": "write artifact.txt"}], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/max-iterations/base_state.json b/examples/openhands-certify/fixtures/conversations/max-iterations/base_state.json new file mode 100644 index 0000000..abdb43a --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/max-iterations/base_state.json @@ -0,0 +1,54 @@ +{ + "id": "00000000-0000-4000-8000-000000000002", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": [ + "FinishTool", + "ThinkTool" + ], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000002", + "max_iterations": 3, + "stuck_detection": true, + "execution_status": "error", + "confirmation_policy": { + "kind": "NeverConfirm" + }, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": { + "secret_sources": {} + }, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..9b47fef --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,20 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [ + { + "cache_prompt": false, + "type": "text", + "text": "write artifact.txt" + } + ], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab01.json b/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab01.json new file mode 100644 index 0000000..5fb8269 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/max-iterations/events/event-00001-b1c2d3e4-0000-4000-8000-00000000ab01.json @@ -0,0 +1,9 @@ +{ + "id": "b1c2d3e4-0000-4000-8000-00000000ab01", + "timestamp": "2026-07-25T12:11:04.510221", + "source": "environment", + "parent_id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "code": "MaxIterationsReached", + "detail": "Agent reached maximum iterations limit (3).", + "kind": "ConversationErrorEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/paused/base_state.json b/examples/openhands-certify/fixtures/conversations/paused/base_state.json new file mode 100644 index 0000000..da8ce38 --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/paused/base_state.json @@ -0,0 +1,54 @@ +{ + "id": "00000000-0000-4000-8000-000000000005", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": [ + "FinishTool", + "ThinkTool" + ], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000005", + "max_iterations": 500, + "stuck_detection": true, + "execution_status": "paused", + "confirmation_policy": { + "kind": "NeverConfirm" + }, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": { + "secret_sources": {} + }, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/paused/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/paused/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..9b47fef --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/paused/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,20 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [ + { + "cache_prompt": false, + "type": "text", + "text": "write artifact.txt" + } + ], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/running/base_state.json b/examples/openhands-certify/fixtures/conversations/running/base_state.json new file mode 100644 index 0000000..f5cf06d --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/running/base_state.json @@ -0,0 +1,54 @@ +{ + "id": "00000000-0000-4000-8000-000000000006", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": [ + "FinishTool", + "ThinkTool" + ], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000006", + "max_iterations": 500, + "stuck_detection": true, + "execution_status": "running", + "confirmation_policy": { + "kind": "NeverConfirm" + }, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": { + "secret_sources": {} + }, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/running/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/running/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..9b47fef --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/running/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,20 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [ + { + "cache_prompt": false, + "type": "text", + "text": "write artifact.txt" + } + ], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/conversations/stuck/base_state.json b/examples/openhands-certify/fixtures/conversations/stuck/base_state.json new file mode 100644 index 0000000..41b8fcd --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/stuck/base_state.json @@ -0,0 +1,54 @@ +{ + "id": "00000000-0000-4000-8000-000000000003", + "agent": { + "llm": { + "model": "gpt-4o-mini", + "api_key": "**********", + "auth_type": "api_key", + "usage_id": "recipe", + "kind": "LLM" + }, + "tools": [], + "mcp_config": {}, + "include_default_tools": [ + "FinishTool", + "ThinkTool" + ], + "kind": "Agent" + }, + "workspace": { + "working_dir": "workspace", + "kind": "LocalWorkspace" + }, + "persistence_dir": "conversations/00000000000040008000000000000003", + "max_iterations": 500, + "stuck_detection": true, + "execution_status": "stuck", + "confirmation_policy": { + "kind": "NeverConfirm" + }, + "stats": { + "usage_to_metrics": { + "recipe": { + "model_name": "gpt-4o-mini", + "accumulated_cost": 0.0141, + "accumulated_token_usage": { + "model": "gpt-4o-mini", + "prompt_tokens": 8123, + "completion_tokens": 412, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, + "context_window": 128000, + "per_turn_token": 0, + "response_id": "" + } + } + } + }, + "secret_registry": { + "secret_sources": {} + }, + "tags": {}, + "agent_state": {} +} diff --git a/examples/openhands-certify/fixtures/conversations/stuck/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json b/examples/openhands-certify/fixtures/conversations/stuck/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json new file mode 100644 index 0000000..9b47fef --- /dev/null +++ b/examples/openhands-certify/fixtures/conversations/stuck/events/event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json @@ -0,0 +1,20 @@ +{ + "id": "5a1d5379-b1d4-4772-9292-7b002555b529", + "timestamp": "2026-07-25T12:10:13.226484", + "source": "user", + "parent_id": null, + "llm_message": { + "role": "user", + "content": [ + { + "cache_prompt": false, + "type": "text", + "text": "write artifact.txt" + } + ], + "thinking_blocks": [] + }, + "activated_skills": [], + "extended_content": [], + "kind": "MessageEvent" +} diff --git a/examples/openhands-certify/fixtures/workspaces/green/artifact.txt b/examples/openhands-certify/fixtures/workspaces/green/artifact.txt new file mode 100644 index 0000000..e935fb3 --- /dev/null +++ b/examples/openhands-certify/fixtures/workspaces/green/artifact.txt @@ -0,0 +1 @@ +hello from openhands diff --git a/examples/openhands-certify/fixtures/workspaces/stale/artifact.txt b/examples/openhands-certify/fixtures/workspaces/stale/artifact.txt new file mode 100644 index 0000000..dd4c378 --- /dev/null +++ b/examples/openhands-certify/fixtures/workspaces/stale/artifact.txt @@ -0,0 +1 @@ +HELLO stub diff --git a/scripts/test_openhands_recipe.py b/scripts/test_openhands_recipe.py new file mode 100644 index 0000000..9baab6b --- /dev/null +++ b/scripts/test_openhands_recipe.py @@ -0,0 +1,205 @@ +"""Acceptance for the OpenHands recipe: a post-run certifier reads a persisted +Conversation record (``base_state.json`` + ``events/``), projects it through +``loop.integrations``, and emits a contract ``loop doctor`` round-trips and +``loop metrics`` scores clean. + +Deterministic and credential-free: every case is driven by a COMMITTED fixture +conversation dir, so this file runs in the default gates matrix on 3.10–3.12. +The live schema-drift alarm against the installed SDK lives in +``test_openhands_sdk_drift.py`` (its own CI job, python 3.12). +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXAMPLE_DIR = REPO_ROOT / "examples" / "openhands-certify" +EXAMPLE = EXAMPLE_DIR / "certify_run.py" +CONVERSATIONS = EXAMPLE_DIR / "fixtures" / "conversations" +WORKSPACES = EXAMPLE_DIR / "fixtures" / "workspaces" + + +def _certify(out_dir: Path, conversation: str, workspace: str = "green") -> subprocess.CompletedProcess: + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) + return subprocess.run( + [ + sys.executable, "-B", str(EXAMPLE), str(out_dir), + "--conversation", str(CONVERSATIONS / conversation), + "--agent-workspace", str(WORKSPACES / workspace), + ], + cwd=out_dir.parent, env=env, capture_output=True, text=True, + ) + + +def _cli(cmd: str, workspace: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", cmd, str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + + +def _terminal(out_dir: Path) -> dict: + return json.loads((out_dir / ".loop" / "terminal_state.json").read_text(encoding="utf-8")) + + +def test_finished_and_green_is_succeeded_doctor_clean_and_metrics_clean(tmp_path): + out = tmp_path / "openhands-run" + proc = _certify(out, "finished", "green") + assert proc.returncode == 0, proc.stdout + proc.stderr + + doctored = _cli("doctor", out) + assert doctored.returncode == 0, doctored.stdout + assert json.loads(doctored.stdout)["ok"] is True + + terminal = _terminal(out) + assert terminal["state"] == "Succeeded" + assert terminal["false_completion"] is False + assert terminal["evidence"] + + metrics = _cli("metrics", out) + assert metrics.returncode == 0, metrics.stdout + metrics.stderr + card = json.loads(metrics.stdout) + assert card["false_completion_rate"] == 0.0 + assert card["false_completions"] == 0 + assert card["iterations_claiming_success"] >= 1 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["unmatched_verify"] == [] + assert prov["unrecognized_outcomes"] == [] + assert prov["fcr_methods_agree"] is True + + +def test_finished_but_withheld_check_red_is_false_completion_never_succeeded(tmp_path): + """Issue #37's pinned invariant: the visible check passes, the withheld one + does not — the run is recorded as a false completion, never laundered.""" + out = tmp_path / "openhands-run-stale" + proc = _certify(out, "finished", "stale") + + terminal = _terminal(out) + assert terminal["state"] == "FailedUnverifiable" + assert terminal["state"] != "Succeeded" + assert terminal["false_completion"] is True, (terminal, proc.stdout, proc.stderr) + assert terminal["reason"] == "visible passed but holdout failed — false completion" + + doctored = _cli("doctor", out) + assert json.loads(doctored.stdout)["ok"] is True # an honest failure is a valid contract + + +@pytest.mark.parametrize( + "conversation,expected", + [ + ("max-iterations", "FailedBudget"), + ("stuck", "FailedBudget"), + ("blocked", "FailedBlocked"), + ("paused", "AbortedByHuman"), + ("running", "FailedUnverifiable"), + ], +) +def test_execution_status_maps_to_typed_terminal(tmp_path, conversation, expected): + """Every non-happy OpenHands terminal maps to its typed state even though the + gate over the workspace is GREEN — the engine signal is never overridden by a + passing check.""" + out = tmp_path / f"openhands-run-{conversation}" + proc = _certify(out, conversation, "green") + assert proc.returncode == 1, proc.stdout + proc.stderr + + terminal = _terminal(out) + assert terminal["state"] == expected + assert terminal["state"] != "Succeeded" + assert json.loads(_cli("doctor", out).stdout)["ok"] is True + + +def test_max_iterations_error_is_budget_not_blocked(tmp_path): + """The precedence trap: MaxIterationsReached arrives AS execution_status + 'error'. Setting both external_error and budget_exhausted would resolve to + FailedBlocked (blocked outranks budget) and silently lose the budget signal — + so the mapper must set exactly one.""" + sys.path.insert(0, str(EXAMPLE_DIR)) + try: + import certify_run + finally: + sys.path.remove(str(EXAMPLE_DIR)) + + record = certify_run.read_conversation(CONVERSATIONS / "max-iterations") + outcome = certify_run.to_engine_outcome(record, ["a.json"]) + assert outcome.budget_exhausted is True + assert outcome.external_error is None + assert outcome.reached_end is False + + blocked = certify_run.to_engine_outcome( + certify_run.read_conversation(CONVERSATIONS / "blocked"), ["a.json"] + ) + assert blocked.external_error is not None + assert blocked.budget_exhausted is False + assert "LLMAuthenticationError" in blocked.external_error + + +def test_error_status_without_an_error_event_still_blocks(tmp_path): + """An 'error' record whose ConversationErrorEvent is missing must not fall + through to a certifiable outcome: external_error is never empty.""" + sys.path.insert(0, str(EXAMPLE_DIR)) + try: + import certify_run + finally: + sys.path.remove(str(EXAMPLE_DIR)) + + conv = tmp_path / "conv" + (conv / "events").mkdir(parents=True) + (conv / "base_state.json").write_text( + json.dumps({"execution_status": "error"}), encoding="utf-8" + ) + outcome = certify_run.to_engine_outcome(certify_run.read_conversation(conv), ["a.json"]) + assert outcome.external_error + assert outcome.budget_exhausted is False + + +def test_events_are_read_in_index_order_not_lexical_order(tmp_path): + """event-{idx:05d} overflows past 99999, where lexical order and index order + disagree — the last error event must still be the last one written.""" + sys.path.insert(0, str(EXAMPLE_DIR)) + try: + import certify_run + finally: + sys.path.remove(str(EXAMPLE_DIR)) + + conv = tmp_path / "conv" + events = conv / "events" + events.mkdir(parents=True) + (conv / "base_state.json").write_text( + json.dumps({"execution_status": "error"}), encoding="utf-8" + ) + for idx, code in ((99999, "LLMAuthenticationError"), (100000, "MaxIterationsReached")): + (events / f"event-{idx:05d}-aaaaaaaa-0000-4000-8000-{idx:012d}.json").write_text( + json.dumps( + { + "id": f"aaaaaaaa-0000-4000-8000-{idx:012d}", + "source": "environment", + "code": code, + "detail": "", + "kind": "ConversationErrorEvent", + } + ), + encoding="utf-8", + ) + record = certify_run.read_conversation(conv) + assert [Path(p).name for p in record["event_paths"]][-1].startswith("event-100000-") + outcome = certify_run.to_engine_outcome(record, ["a.json"]) + assert outcome.budget_exhausted is True + assert outcome.external_error is None + + +def test_certifier_imports_no_openhands_package(): + """The certifier is a stdlib reader over a documented on-disk layout — that is + what keeps it on the 3.10 floor while the SDK requires 3.12.""" + lines = [line.strip() for line in EXAMPLE.read_text(encoding="utf-8").splitlines()] + assert not [ + line for line in lines + if line.startswith(("import openhands", "from openhands")) + ] diff --git a/scripts/test_openhands_sdk_drift.py b/scripts/test_openhands_sdk_drift.py new file mode 100644 index 0000000..b8d5d88 --- /dev/null +++ b/scripts/test_openhands_sdk_drift.py @@ -0,0 +1,75 @@ +"""Live schema-drift alarm for the OpenHands recipe. + +``base_state.json`` is a ``model_dump_json()`` of a fast-moving pydantic model, +and ``ConversationErrorEvent.code`` is a free-form ``str`` — ``"MaxIterationsReached"`` +is a source literal, not a contract. So the recipe's committed fixtures are pinned +against the INSTALLED SDK here: if OpenHands renames a constant, drops an execution +status, or stops emitting that error code, this fires. + +Skipped everywhere the SDK is absent (it requires python >=3.12); the behavioural +e2e in ``test_openhands_recipe.py`` is fixture-driven and needs no install. +""" + +from __future__ import annotations + +import inspect +import json +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("openhands.sdk") + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXAMPLE_DIR = REPO_ROOT / "examples" / "openhands-certify" +CONVERSATIONS = EXAMPLE_DIR / "fixtures" / "conversations" + +sys.path.insert(0, str(EXAMPLE_DIR)) +import certify_run # noqa: E402 + + +def test_persistence_constants_match_the_certifier(): + from openhands.sdk.conversation import persistence_const as pc + + assert pc.BASE_STATE == certify_run.BASE_STATE + assert pc.EVENTS_DIR == certify_run.EVENTS_DIR + assert pc.EVENT_FILE_PATTERN == "event-{idx:05d}-{event_id}.json" + assert pc.EVENT_NAME_RE.match("event-00000-5a1d5379-b1d4-4772-9292-7b002555b529.json") + + +def test_every_fixture_status_is_a_live_execution_status(): + from openhands.sdk.conversation.state import ConversationExecutionStatus + + live = {member.value for member in ConversationExecutionStatus} + assert {"finished", "error", "stuck", "paused", "running", "idle"} <= live + + for base_state in sorted(CONVERSATIONS.glob("*/base_state.json")): + status = json.loads(base_state.read_text(encoding="utf-8"))["execution_status"] + assert status in live, (base_state, status) + + +def test_error_event_still_serializes_the_shape_the_mapper_reads(): + from openhands.sdk.event.conversation_error import ConversationErrorEvent + + event = json.loads( + ConversationErrorEvent( + source="environment", code="MaxIterationsReached", detail="…" + ).model_dump_json() + ) + assert event["kind"] == certify_run.ERROR_EVENT_KIND + assert event["code"] == certify_run.MAX_ITERATIONS_CODE + assert "detail" in event + + +def test_max_iterations_code_is_still_the_run_loop_literal(): + from openhands.sdk.conversation.impl import local_conversation + + assert certify_run.MAX_ITERATIONS_CODE in inspect.getsource(local_conversation) + + +def test_fixture_records_still_read_through_the_certifier(): + for conv_dir in sorted(p.parent for p in CONVERSATIONS.glob("*/base_state.json")): + record = certify_run.read_conversation(conv_dir) + assert record["state"]["execution_status"] + assert record["event_paths"]