diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0966ca..e7cca5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,23 @@ jobs: - name: Install recipe dependencies run: python -m pip install --upgrade pip pyyaml pytest jsonschema langgraph - name: LangGraph recipe end-to-end - run: python -B -m pytest -q -p no:cacheprovider scripts/test_langgraph_recipe.py + run: python -B -m pytest -q -p no:cacheprovider scripts/test_langgraph_recipe.py scripts/test_langgraph_recipe_st3.py + + recipe-temporal: + name: recipe (temporal) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install recipe dependencies + run: python -m pip install --upgrade pip pyyaml pytest jsonschema temporalio + - name: Temporal recipe end-to-end + # start_local() downloads the Temporal dev-server binary (~20MB) to + # $TMPDIR/temporal-sdk-python- — ephemeral and quick, so (like + # recipe-langgraph) this job carries no cache step. + run: python -B -m pytest -q -p no:cacheprovider scripts/test_temporal_recipe.py action-dogfood: name: action (dogfood on flagship example) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80a2d75..896851b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,24 @@ All notable changes to `loop-engineer` are documented here. `WORKFLOW.md` and `README.md` are reworded to describe the mechanism; the 0.3.4 history is left intact. +## Unreleased + +**ST3 — integration adapters.** `loop/integrations.py`: an engine-neutral, +pure-stdlib projection (`EngineOutcome` + `to_terminal_state`) from any +engine's "the run ended" signal onto the 7 typed terminal states, with the +fixed precedence safety → human → blocked → budget → spec-gap → gate verdict. +`Succeeded` is reachable only through a green `holdout_gate.decide` verdict, +a clean anticheat sweep, a met criterion, and evidence; `false_completion` is +copied from the gate, never synthesized; missing gate/anticheat input fails +closed to `FailedUnverifiable`. The LangGraph recipe is upgraded in place to +this bar (its run now scores clean under `loop metrics` — closing the +recorded FCR-1.0 follow-up) and a Temporal recipe lands +(`examples/temporal-certify/`, certify-activity pattern, cancellation → +`AbortedByHuman`, retry exhaustion → `FailedBlocked`, timeout → +`FailedBudget`). Both recipes pin the false-completion invariant +(visible-green/holdout-red → `FailedUnverifiable` with +`false_completion: true`, never `Succeeded`) and pass the doctor round-trip. + ## 0.7.0 — 2026-07-08 **ST2 — the portable standard.** The on-disk contract is now a documented, diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..0b4aa92 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,125 @@ +# CLAUDE.md — loop-engineer + +Read this first. It replaces the "continue where we left off" cold-start that has been +booting this repo for 9+ days. Nested git repo (`SollanSystems/loop-engineer`, MIT) — +do NOT commit CLAUDE.md or treat it as plugin content. + +## What this is + +A Claude-Code plugin that designs, launches, verifies, repairs, and improves **agent +loops**. It is an **architect + operator**, not a task-doer: it stands up the operating +contract for a long-running agentic-coding run, gates it against false completion, and +mines its own history for compounding improvement. + +Two shipping surfaces from one repo: +- **Plugin** — hub-and-spoke skill suite (router + 8 spokes) invoked inside Claude Code. +- **Portable Loop Contract Core** — a pure-stdlib Python package `loop` + CLI + (`loop` / `loop-engineer`, also `uvx loop-engineer`) that validates and inspects + repo-native operating contracts on any stack. `pip install -e .` pulls **zero** runtime + deps; two optional extras enrich validation: `yaml` (PyYAML) and `schemas` (jsonschema). + +## Skill architecture (9 skills = router + 8 spokes) + +| Skill | Role | +|---|---| +| `loop-engineer` | Router — dispatches broad agent-loop intent to the right spoke. | +| `loop-architect` | The brain — classify the task, choose architecture + Claude-Code realization, emit a structured ADR (architecture, loop patterns, risk profile, terminal-state plan, next spokes). | +| `loop-contract` | Scaffold the repo-OS contract (SPEC / WORKFLOW / TASKS.json / RUNLOG / `.loop/state.json` + `verify-*` skeletons) from an ADR, then run the pre-execution reflection. | +| `loop-run` | The operator — run/resume the state machine one bounded transition at a time; every run ends in exactly one of the 7 terminal states. | +| `loop-repair` | Patch-and-repair loop — classify the failure mode, make the smallest bounded repair, write a 7-field repair record, enforce a max-attempt cap; refuses to widen scope or edit tests to fake a pass. | +| `loop-evals` | Design the eval harness — the 7-layer suite, FCR + repair-productivity as first-class metrics, deterministic-first-then-rubric, judge calibration. Delegates the deterministic gate to the contract's own `verify-*` scripts. | +| `loop-flywheel` | Turn run history (RUNLOG / traces / receipts) into new eval cases; drive baseline→harden→regression→freeze; compact memory. | +| `loop-inspector` | Read-only score of a **foreign** loop against the prime-directive checklist + 7 terminal states → a scored gap report (advisory heuristic, not a gate). | +| `loop-runtime-monitor` | Watch a running loop from outside — detect stall / repair-churn / budget-overrun and surface one intervention. | + +## Version + state + +- **Current release: v0.7.0** (`.claude-plugin/plugin.json` + `pyproject.toml`), main at `4647820`. +- **Active branch: `feat/v0.8.0-composes-the-field`** (v0.8.0 NOT yet released — release cut + is the last commit of PR-B). Working tree has in-progress Temporal recipe files (Task 3). +- **Shipped through v0.7.0:** + - ST2 "Portable standard" — `reference/repo-os-contract.md` promoted to the normative, + versioned, tool-agnostic spec (PR #31). + - External-review credibility patch set **PRs #27–#30** (10 findings): empty-evidence + terminal now fails doctor; terminal write-once + atomic writes; strict UTF-8 + declared + ledger discovery; strict-by-install on the Action + pre-commit hook; verify-script + existence checks; **inspector honesty** (#30) — keyword-stuffed fakes no longer score + 100/strong, `loop inspect` documented as advisory while `loop doctor` is the hard gate. + - Adoption slices (PRs #19–#23): PyPI wheel substrate, `loop/emit.py` writer API + + LangGraph recipe, `hooks/stop_firewall.py` Stop-hook false-completion firewall, + composite `action.yml` + `.pre-commit-hooks.yaml`. +- **v0.8.0 "composes-the-field" (in progress):** ST3 integration adapters + (`loop/integrations.py` EngineOutcome→terminal projection, LangGraph recipe upgraded to + gate+adapter+metrics-clean, Temporal recipe) + ST4 contributor funnel (foreign-harness + inspect adapter `loop/foreign.py`, gap report, `flaky-test-triage` example, contributor + issue drafts + CONTRIBUTING). Resume ledger: `.superpowers/sdd/` (progress.md + briefs). +- **Test baseline: 395 tests** collected on the v0.8.0 branch (372/10 at v0.7.0; ~10 skip + in structural-fallback mode without jsonschema). +- **Dogfood:** the suite has been run on its own v1.0 launch (self-hosted contract at + `roadmap/launch/`) and on `examples/coverage-repair` (doctor-clean, inspect 90/strong). + +## Load-bearing invariants (enforced by `scripts/self_eval.py` vs `evals/cases/structural.json`) + +- **7 canonical terminal states:** `Succeeded`, `FailedUnverifiable`, `FailedBlocked`, + `FailedBudget`, `FailedSafety`, `FailedSpecGap`, `AbortedByHuman`. +- **7-field repair record:** `failure_mode`, `hypothesis`, `repair_action`, + `verification_before`, `verification_after`, `remaining_delta`, `productive`. +- **7-layer eval suite:** deterministic-correctness, artifact-quality, human-calibration, + loop-behavior, security/governance, regression-resistance, cost/efficiency. +- **2 first-class metrics:** `false-completion-rate` (FCR), `repair-productivity` (RP). +- Also pinned: 6-item `failure_mode_taxonomy`, `repair_cap_default` 2, `rubric_target_mean` + 9.5 (advisory), 9 skill names, 8 reference filenames, 14 template filenames, MIT license. +- **When the suite changes** (new skill/template/reference/terminal state), update + `evals/cases/structural.json` — the self-eval checks compare live repo state against it. + +## Verify commands (run from the repo root; this env has no system pytest — use `uv run`) + +```bash +uv run --with pyyaml python3 -B scripts/validate_frontmatter.py # 9 SKILL.md frontmatter blocks +uv run --with pyyaml python3 -B scripts/self_eval.py # structural invariants (self-locates root) +uv run --with pyyaml --with jsonschema --with pytest python3 -B -m pytest -q -p no:cacheprovider scripts # full suite +``` + +CLI subcommands (`python3 -m loop ` — commands: `scaffold doctor validate verify +inspect metrics`): `loop doctor` is the **hard gate**, `loop inspect .` is the **advisory +scorecard**. CI (`.github/workflows/ci.yml`) installs `pyyaml pytest jsonschema` and runs +`python -B -m pytest -q -p no:cacheprovider scripts`; without jsonschema the core falls back +to structural hand-checks (a deliberate design, not a bug). + +## Install / refresh + +Installed **user-scope** from the local marketplace `loop-engineer-local`. The plugin-cache +copy is a **static COPY** (nested under `cache//loop-engineer//`), NOT a symlink — +it goes **stale after any post-install commit**. Refresh with: + +```bash +git -C /mnt/c/Dev/projects/loop-engineer archive HEAD | tar -x -C +diff -rq /mnt/c/Dev/projects/loop-engineer # verify +``` + +Then **restart Claude Code** to reload the skills. + +## Roadmap / open work + +- **Human gates (blocking PyPI):** register the PyPI pending trusted publisher + (project `loop-engineer`, owner `SollanSystems`, workflow `publish.yml`, environment + `pypi`), then `git tag v0.7.0 4647820 && git push origin v0.7.0`; verify the funnel + `uvx loop-engineer@0.7.0 inspect .` from a scratch dir. +- **Finish v0.8.0:** Temporal recipe (Task 3) → ship PR-A → PR-B (foreign adapter, gap + report, flaky example, contributor funnel, 0.8.0 release cut) → file the 6 contributor + issues at merge. +- **Then:** "inspect N public harnesses" scoreboard post; Show HN launch (human-gated). +- **Positioning (verified):** never claim to coin "loop engineering" — cobusgreyling's + `loop-engineering` (~4.6k stars) owns the term. Own **"false completion"** / + **"proof-of-done contract"** instead. + +## Repo conventions + +- Specs → `docs/superpowers/specs/` (dated); plans → `docs/superpowers/plans/` (dated). +- Session narratives → `memory/session-summaries/YYYY-MM-DD-*.md` (memory/ is gitignored). +- **Gitignored workbench/telemetry** (not plugin content): `.loop/`, `review/`, `roadmap/`, + `.claude/`, `.gsd/`, `memory/`, `.tmp/`. CI runs on a fresh checkout where these are + absent — never point a CI/dogfood job at the live gitignored `.loop/`; target a tracked + `examples/*` contract instead. +- Env quirks: the Bash deny-list blocks `rm`, bare `cd`, `VAR=` assignments, `timeout`, + `printf`, `source` — use `git -C `, literal absolute paths, and `bash -c`. diff --git a/docs/integrations/langgraph.md b/docs/integrations/langgraph.md index a42edbb..7aadeec 100644 --- a/docs/integrations/langgraph.md +++ b/docs/integrations/langgraph.md @@ -1,30 +1,88 @@ -# LangGraph — proof-of-done in 10 lines +# LangGraph — gate the graph, then emit proof-of-done -`loop.emit` is a pure-stdlib writer: your graph keeps its own runtime, and the -terminal node records evidence-backed state the `loop` CLI can independently -validate. `pip install loop-engineer` (LangGraph itself stays your dependency). +LangGraph owns the ORCHESTRATE tier: it stays your runtime — the state machine +that routes nodes, holds state, and decides what runs next. Loop Engineer adds +the tier *above* it — a contract-and-proof layer that turns "the graph reached +`END`" into evidence-backed, independently-checkable proof-of-done. It never +replaces LangGraph; it certifies what LangGraph produced. + +## The pattern + +Make a `certify` node the **only** edge into `END`. It runs the same visible + +withheld-holdout split the loop optimized against through the real gate +(`holdout_gate.decide`) and the trajectory sweep (`anticheat_scan.scan`), +projects the result through `to_terminal_state`, and records it via `loop.emit` +— which refuses a dishonest `Succeeded` before anything hits disk. ```python from loop import emit +from loop.integrations import EngineOutcome, to_terminal_state + +def certify(state): # the ONLY node wired to END + gate = holdout_gate.decide(visible, holdout) # visible green + holdout green? + ac = anticheat_scan.scan(diff_text="", trajectory=[...]) + terminal = to_terminal_state( + outcome=EngineOutcome(reached_end=True, artifacts=[...]), + gate_verdict=gate, anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + emit.terminate(ws, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1) + return {"terminal_state": terminal["state"]} +``` -emit.open_contract("run/") # once, before the graph runs +Wire it so nothing else reaches `END`: -def conclude(state): # your graph's terminal node - emit.append_iteration("run/", iteration_id=1, outcome="task_passed", - task_id="T1", verify_cmd="pytest -q", verify_outcome="pass") - emit.terminate("run/", state="Succeeded", - criteria_met={"tests": True}, evidence=["reports/pytest.txt"]) - return {} +```python +graph.add_edge("do_work", "certify").add_edge("certify", END) ``` -`emit.terminate` **refuses an evidence-free `Succeeded`** (raises `EmitError`) — -the same cross-check `loop doctor` enforces, applied before the file exists. +## LangGraph signal → typed terminal state + +| LangGraph signal | Typed terminal state | +|---|---| +| graph reached `END`, holdout green + anticheat clean | `Succeeded` | +| graph reached `END`, visible green / holdout red | `FailedUnverifiable` (`false_completion: true`) | +| `GraphRecursionError` (LangGraph's own step cap) | `FailedBudget` | +| caught tool/credential exception | `FailedBlocked` | +| operator interrupt | `AbortedByHuman` | + +## Zero-install mode -Gate it in CI: +The `loop.integrations` module is convenience, not a requirement — the whole +projection is ~15 lines you can paste into any graph with no dependency: + +```python +def to_terminal(gate, anticheat, criteria_met, evidence, + *, human_abort=False, blocked=None, over_budget=False): + fc = gate.get("false_completion") is True + if anticheat.get("downgrade_to") == "FailedSafety": state = "FailedSafety" + elif human_abort: state = "AbortedByHuman" + elif blocked: state = "FailedBlocked" + elif over_budget: state = "FailedBudget" + elif any(v is None for v in criteria_met.values()): state = "FailedSpecGap" + elif (not gate or not anticheat or anticheat.get("downgrade_to") + or gate.get("verdict") != "Succeeded" or fc + or not any(criteria_met.values()) or not evidence): state = "FailedUnverifiable" + else: state = "Succeeded" + return {"schema": "loop-engineer/terminal@1", "state": state, + "criteria_met": {k: v is True for k, v in criteria_met.items()}, + "evidence": list(evidence), "false_completion": fc} +``` + +## Gate it in CI ```yaml - run: pip install loop-engineer -- run: loop doctor run/ +- run: loop doctor run/ # -> {"ok": true}: the contract is structurally honest +- run: loop metrics run/ # -> false_completion_rate + evidence-backed scorecard ``` -Full runnable example: [`examples/langgraph-emit/`](../../examples/langgraph-emit/). +`loop metrics` scores the run from its on-disk evidence (RUNLOG success claims, +the verify bundle, the held-out verdict) — not from the graph's narration. + +Verified against `langgraph` 1.2.8 (2026-07-08). + +Full runnable example (happy path + `--sabotage-holdout` false-completion demo): +[`examples/langgraph-emit/`](../../examples/langgraph-emit/). diff --git a/docs/integrations/temporal.md b/docs/integrations/temporal.md new file mode 100644 index 0000000..c7e7753 --- /dev/null +++ b/docs/integrations/temporal.md @@ -0,0 +1,90 @@ +# Temporal — durable execution below, proof-of-done above + +Temporal owns the EXECUTE tier: your durable runtime — the workflow *survives +crashes*, retries activities, resumes where it left off. What it says nothing +about is whether the work is *correct*. Loop Engineer adds the tier *above* it — +a contract-and-proof layer that turns "the workflow returned" into evidence-backed +proof-of-done. It never replaces Temporal; it certifies what Temporal ran. + +## The pattern + +A `certify` **activity** is the workflow's only path to a returned result: +activities do the I/O (write files, run the gate), the workflow stays +deterministic and just orchestrates. The certify activity runs the same visible ++ withheld-holdout split the loop optimized against through the real gate +(`holdout_gate.decide`) and trajectory sweep (`anticheat_scan.scan`), projects it +through `to_terminal_state`, and records it via `loop.emit` — which refuses a +dishonest `Succeeded` before anything hits disk. + +```python +@activity.defn +async def certify_activity(args: WorkArgs) -> dict: + gate = holdout_gate.decide(visible, holdout) # visible green + holdout green? + ac = anticheat_scan.scan(diff_text="", trajectory=[...]) + terminal = to_terminal_state( + outcome=EngineOutcome(reached_end=True, artifacts=[...]), + gate_verdict=gate, anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + emit.terminate(ws, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1) + return {"state": terminal["state"]} + +@workflow.defn +class CertifiedGoalWorkflow: + @workflow.run + async def run(self, args: WorkArgs) -> dict: + await workflow.execute_activity(do_work_activity, args, ...) + return await workflow.execute_activity(certify_activity, args, ...) # the ONLY return path +``` + +Failures short of the certify activity are mapped host-side off `WorkflowFailureError.cause`, so a crash/cancel/timeout still lands an honest terminal. + +## Temporal signal → typed terminal state + +| Temporal signal | Typed terminal state | +|---|---| +| workflow returned via certify activity, holdout green + anticheat clean | `Succeeded` | +| workflow returned, visible green / holdout red | `FailedUnverifiable` (`false_completion: true`) | +| workflow `CancelledError` | `AbortedByHuman` | +| activity `RetryPolicy` exhaustion on an external dependency | `FailedBlocked` | +| workflow timeout (`run_timeout`) | `FailedBudget` | + +## Zero-install mode + +The `loop.integrations` module is convenience, not a requirement — the whole +projection is the SAME ~15 lines you can paste into any engine (the adapter is +engine-neutral; byte-identical to the LangGraph recipe's): + +```python +def to_terminal(gate, anticheat, criteria_met, evidence, + *, human_abort=False, blocked=None, over_budget=False): + fc = gate.get("false_completion") is True + if anticheat.get("downgrade_to") == "FailedSafety": state = "FailedSafety" + elif human_abort: state = "AbortedByHuman" + elif blocked: state = "FailedBlocked" + elif over_budget: state = "FailedBudget" + elif any(v is None for v in criteria_met.values()): state = "FailedSpecGap" + elif (not gate or not anticheat or anticheat.get("downgrade_to") + or gate.get("verdict") != "Succeeded" or fc + or not any(criteria_met.values()) or not evidence): state = "FailedUnverifiable" + else: state = "Succeeded" + return {"schema": "loop-engineer/terminal@1", "state": state, + "criteria_met": {k: v is True for k, v in criteria_met.items()}, + "evidence": list(evidence), "false_completion": fc} +``` + +## Gate it in CI + +```yaml +- run: pip install loop-engineer +- run: loop doctor run/ # -> {"ok": true}: the contract is structurally honest +- run: loop metrics run/ # -> false_completion_rate + evidence-backed scorecard +``` + +`loop metrics` scores the run from its on-disk evidence — not the workflow's narration. + +Verified against `temporalio` 1.30.0 (2026-07-08). + +Full runnable example (happy path + `--sabotage-holdout` demo + cancellation → `AbortedByHuman`): [`examples/temporal-certify/`](../../examples/temporal-certify/). diff --git a/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md b/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md new file mode 100644 index 0000000..5cccd3f --- /dev/null +++ b/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md @@ -0,0 +1,2114 @@ +# v0.8.0 "Composes the field" (ST3 + ST4) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship the whole v0.8.0 milestone — ST3 (a pure-stdlib engine-outcome→typed-terminal adapter + LangGraph and Temporal recipes) and ST4 (a read-only foreign-harness inspect adapter + gap report, a 2nd runnable example showcasing RP, a contributor funnel, and the 0.8.0 release cut) — as two PRs. + +**Architecture:** PR-A adds `loop/integrations.py` (a pure projection: `EngineOutcome` + `to_terminal_state` implementing the fixed precedence *safety → human → blocked → budget → spec-gap → gate verdict*), upgrades `examples/langgraph-emit/` in place to route its terminal through the gate + adapter + `loop.emit`, and adds a new `examples/temporal-certify/` recipe. PR-B adds `loop/foreign.py` (a Superpowers-layout→`LoopPaths` mapper plugged into `inspect_loop`'s existing path-resolution seam, scorer untouched), a vendored fixture + checked-in gap report, `examples/flaky-test-triage/`, six drafted contributor issues + CONTRIBUTING funnel, and the 0.8.0 release cut as the last commit. + +**Tech Stack:** Pure-stdlib Python ≥3.10 for everything shipped; `langgraph` and `temporalio` are test/CI-only dependencies of their recipes (env-guarded via `pytest.importorskip`); GitHub Actions CI. + +**Spec:** `docs/superpowers/specs/2026-07-08-v0.8.0-composes-the-field-design.md` (approved). Normative for mapping/precedence: `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md`. + +## Global Constraints + +- **Two PRs:** PR-A = ST3 (Tasks 1–4), PR-B = ST4 + release cut (Tasks 5–10). PR-B branches from main after PR-A merges. The 0.8.0 release commit is the **last** commit of PR-B. +- **Paths are the shipped reality, not the 2026-06-30 spec's:** helper = `loop/integrations.py` (NOT `loop_engineer.integrations`); docs = `docs/integrations/.md` (NOT `reference/integrations/`). +- **All disk writes go through `loop.emit`** (`open_contract` / `append_iteration` / `append_receipt` / `terminate`). No second write path for contract artifacts. (Recipes may write their own verify-bundle/verdict JSONs under `.loop/artifacts/` — those are evidence artifacts, not contract objects.) *Adjudicated carve-out (PR-A whole-branch review, 2026-07-08):* both recipes reset `/RUNLOG.md` to emit's own fresh header immediately after `open_contract`, because the scaffold seeds an unfilled `{{ITERATION_OUTCOME}}` placeholder that `loop metrics` flags as an unrecognized outcome token and the root fix (templates/scaffold/emit) is outside PR-A's file discipline. The reset strips a placeholder, never fabricates state. Root-fix affordance (e.g. `emit.open_contract(seed_runlog=False)`) is tracked as a PR-B contributor issue. +- **PR-A file discipline:** `loop/` gains ONLY `integrations.py`. No modification to `scripts/holdout_gate.py`, `scripts/anticheat_scan.py`, `scripts/inspect_loop.py`, `scripts/metrics.py`, `scripts/self_eval.py`, `scripts/validate_frontmatter.py`, or anything under `schemas/`, `templates/`, `evals/`. `scripts/test_langgraph_recipe.py` stays **byte-identical** (new assertions go in a NEW file `scripts/test_langgraph_recipe_st3.py`). New test files under `scripts/` are additions, allowed. +- **PR-B scorer discipline:** `scripts/inspect_loop.py` may change ONLY (a) the path-resolution seam (3 `resolve_loop_paths(...)` call sites → `_resolve_paths(...)`) and (b) additive report labeling (`foreign_layout`, `advisory`). Zero changes to `_CHECKS` weights, credit tiers, signal regexes, the score cap, or `_verdict`. A regression test pins that the fixture earns no invoked credit. +- **Fixed precedence** in `to_terminal_state`: safety → human → blocked → budget → spec-gap → gate verdict. `FailedSafety` and `AbortedByHuman` beat a green gate. +- **`Succeeded` is reachable only via:** gate `verdict == "Succeeded"` AND anticheat clean of HIGH/CRITICAL AND ≥1 true criterion AND non-empty evidence AND `reached_end`. `false_completion` is **copied** from the gate dict (`gate.get("false_completion") is True`), never synthesized. Missing/empty/structurally-invalid gate or anticheat input → `FailedUnverifiable` (fail closed). +- **`loop/integrations.py` imports:** stdlib only (dataclasses). Zero engine imports, zero `scripts/` imports, zero `loop.*` imports needed. Gate/anticheat results arrive as plain dicts. +- **Complement framing** in every recipe doc and the gap report; never claim to replace an engine. Gap-report factual claims restricted to the vendored fixture (no version-general claims about Superpowers). +- **Library research discipline:** before finalizing each recipe snippet, verify the engine API via Context7 (`/temporalio/sdk-python`; resolve LangGraph fresh) and record a "verified against vX" note in the recipe doc (version = what CI installs, read via `pip show`). +- **Tests must not require pytest-asyncio** — Temporal tests wrap async bodies in `asyncio.run(...)` inside sync `def test_*`. +- **Canonical local test command:** `uv run --with pytest --with pyyaml --with jsonschema python -B -m pytest -q -p no:cacheprovider scripts` (add `--with langgraph` / `--with temporalio` to exercise the env-guarded recipe tests locally). Green baseline before this work: 372 passed / 10 skipped (jsonschema lane). +- **Full gate set** (must be green before each PR): `validate_frontmatter.py`, `self_eval.py`, full pytest, `py_compile loop/*.py scripts/*.py`, `python3 -m loop doctor examples/coverage-repair`, `python3 -m loop inspect examples/coverage-repair`. +- Conventional commits; CHANGELOG under `## Unreleased` in PR-A, converted to `## 0.8.0` by the release-cut task. +- No `[[wikilinks]]` in any new doc outside `skills/` (self_eval resolves wikilinks against the 9 skills). +- Python 3.10 compatibility: no `Self`, no `tomllib`, no dataclass `slots=True` reliance on 3.10+ semantics beyond what the repo already uses. + +--- + +## PR-A — ST3: adapter + two recipes + +### Task 1: `loop/integrations.py` — `EngineOutcome` + `to_terminal_state` + +**Files:** +- Create: `loop/integrations.py` +- Test: `scripts/test_integrations.py` + +**Interfaces:** +- Produces: `EngineOutcome(reached_end: bool, external_error: str | None = None, budget_exhausted: bool = False, human_abort: bool = False, artifacts: Sequence[str] = ())` — frozen dataclass, `artifacts` normalized to `tuple[str, ...]`. +- Produces: `to_terminal_state(outcome: EngineOutcome, gate_verdict: dict | None, anticheat: dict | None, criteria_met: dict[str, bool | None]) -> dict` — returns a `terminal@1` body: `{"schema": "loop-engineer/terminal@1", "state": , "criteria_met": {str: bool}, "evidence": [str], "false_completion": bool, "reason": str}`. A `None` criteria value means "criterion has no mapped check" → `FailedSpecGap`; `None` coerces to `False` in the returned body (unproven ≠ met). +- Consumed by Tasks 2 and 3: recipes splat the body into `emit.terminate(ws, state=t["state"], criteria_met=t["criteria_met"], evidence=t["evidence"], false_completion=t["false_completion"], reason=t["reason"], iteration_id=N)`. + +- [ ] **Step 1.1: Write the failing test** + +Create `scripts/test_integrations.py`: + +```python +"""ST3 adapter contract: the projection from an engine outcome + gate/anticheat +evidence to one of the 7 typed terminal states. Pins the fixed precedence +(safety -> human -> blocked -> budget -> spec-gap -> gate verdict), the +false-completion invariant, and the structural unreachability of Succeeded +without a green gate + clean anticheat + a met criterion + evidence.""" + +from __future__ import annotations + +import importlib.util +import re +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from loop import emit # noqa: E402 +from loop.integrations import EngineOutcome, to_terminal_state # noqa: E402 + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, Path(__file__).parent / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _gate(visible_pass: bool, holdout_pass: bool) -> dict: + """A REAL holdout_gate.decide(...) verdict — never a hand-typed stub.""" + holdout_gate = _load("holdout_gate") + return holdout_gate.decide( + visible=[{"id": "v1", "passed": visible_pass}], + holdout=[{"id": "h1", "passed": holdout_pass}], + ) + + +_CLEAN_AC = {"findings": [], "clean": True, "downgrade_to": None} +_HIGH_AC = { + "findings": [{"signature": "hidden-answer-read", "severity": "high", "evidence": "x"}], + "clean": False, + "downgrade_to": "FailedUnverifiable", +} +_CRITICAL_AC = { + "findings": [{"signature": "gate-tampering", "severity": "critical", "evidence": "x"}], + "clean": False, + "downgrade_to": "FailedSafety", +} +_ENDED = dict(reached_end=True, artifacts=["a.txt"]) + + +def test_succeeded_via_green_gate_clean_anticheat_met_criterion(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True}) + assert body["state"] == "Succeeded" + assert body["false_completion"] is False + assert body["schema"] == "loop-engineer/terminal@1" + assert body["evidence"] == ["a.txt"] + + +def test_false_completion_invariant_visible_green_holdout_red(): + gate = _gate(True, False) + assert gate["false_completion"] is True # the real decide() flag + body = to_terminal_state(EngineOutcome(**_ENDED), gate, _CLEAN_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + assert body["state"] != "Succeeded" + assert body["false_completion"] is True # copied from the gate, not synthesized + + +def test_anticheat_critical_beats_green_gate(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CRITICAL_AC, {"1": True}) + assert body["state"] == "FailedSafety" + + +def test_human_abort_beats_green_gate(): + body = to_terminal_state( + EngineOutcome(reached_end=True, human_abort=True, artifacts=["a"]), + _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert body["state"] == "AbortedByHuman" + + +def test_external_error_maps_to_failed_blocked(): + body = to_terminal_state( + EngineOutcome(reached_end=False, external_error="credential missing"), + {}, {}, {"1": False}, + ) + assert body["state"] == "FailedBlocked" + assert "credential missing" in body["reason"] + + +def test_budget_exhausted_maps_to_failed_budget(): + body = to_terminal_state( + EngineOutcome(reached_end=False, budget_exhausted=True), {}, {}, {"1": False}, + ) + assert body["state"] == "FailedBudget" + + +def test_unmapped_criterion_maps_to_failed_spec_gap(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True, "2": None}) + assert body["state"] == "FailedSpecGap" + assert body["criteria_met"] == {"1": True, "2": False} # None coerces to False + + +def test_all_seven_states_reachable(): + reached = { + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, False), _CLEAN_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(reached_end=False, external_error="x"), {}, {}, {"1": False})["state"], + to_terminal_state(EngineOutcome(reached_end=False, budget_exhausted=True), {}, {}, {"1": False})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CRITICAL_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": None})["state"], + to_terminal_state(EngineOutcome(reached_end=False, human_abort=True), {}, {}, {"1": False})["state"], + } + assert reached == { + "Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", + "FailedSafety", "FailedSpecGap", "AbortedByHuman", + } + + +def test_missing_gate_or_anticheat_fails_closed(): + assert to_terminal_state(EngineOutcome(**_ENDED), None, _CLEAN_AC, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), {}, _CLEAN_AC, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), None, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), {}, {"1": True})["state"] == "FailedUnverifiable" + + +def test_succeeded_unreachable_without_met_criterion_or_evidence(): + no_criterion = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": False}) + assert no_criterion["state"] == "FailedUnverifiable" + no_evidence = to_terminal_state( + EngineOutcome(reached_end=True, artifacts=[]), _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert no_evidence["state"] == "FailedUnverifiable" + not_ended = to_terminal_state( + EngineOutcome(reached_end=False, artifacts=["a"]), _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert not_ended["state"] == "FailedUnverifiable" + + +def test_anticheat_high_downgrades_a_green_gate(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _HIGH_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + + +def test_not_ready_gate_cannot_certify(): + holdout_gate = _load("holdout_gate") + gate = holdout_gate.decide(visible=[], holdout=[]) # NotReady + body = to_terminal_state(EngineOutcome(**_ENDED), gate, _CLEAN_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + + +def test_body_feeds_emit_terminate_round_trip(tmp_path): + ws = tmp_path / "run" + emit.open_contract(ws) + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True}) + path = emit.terminate( + ws, state=body["state"], criteria_met=body["criteria_met"], evidence=body["evidence"], + false_completion=body["false_completion"], reason=body["reason"], iteration_id=1, + ) + assert path.is_file() + + +def test_module_imports_no_engine_and_no_scripts(): + source = (_REPO / "loop" / "integrations.py").read_text(encoding="utf-8") + imports = [l for l in source.splitlines() if re.match(r"\s*(import|from)\s", l)] + for line in imports: + assert "langgraph" not in line and "temporalio" not in line and "scripts" not in line, line + # pure stdlib: the only allowed import roots + for line in imports: + assert re.match(r"\s*(from\s+(__future__|dataclasses|typing)\s+import|import\s+(dataclasses|typing))", line), line +``` + +- [ ] **Step 1.2: Run it to make sure it fails** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_integrations.py` +Expected: FAIL/ERROR with `ModuleNotFoundError: No module named 'loop.integrations'` + +- [ ] **Step 1.3: Implement `loop/integrations.py`** + +```python +"""Engine-outcome -> typed-terminal projection (ST3). + +A pure projection, never a runtime: recipes adapt their engine's native result +into an ``EngineOutcome``, pass the holdout-gate and anticheat results through +as plain dicts (``scripts/holdout_gate.py decide(...)`` / ``scripts/ +anticheat_scan.py scan(...)`` JSON), and this module assembles the +``terminal@1`` body. Every disk write stays in ``loop.emit``. + +The fixed precedence — safety -> human -> blocked -> budget -> spec-gap -> +gate verdict — means a gamed (FailedSafety) or human-killed (AbortedByHuman) +run can never launder itself into Succeeded. ``Succeeded`` is reachable ONLY +via a green gate verdict + anticheat clean of HIGH/CRITICAL findings + at +least one met criterion + non-empty evidence. ``false_completion`` is copied +from the gate result, never synthesized. Missing or structurally-empty gate/ +anticheat input fails closed to ``FailedUnverifiable`` — the same posture as +``holdout_gate`` on an empty holdout set. + +Pure stdlib; imports no engine package and nothing from ``scripts/`` — so +installing this helper never pulls LangGraph/Temporal/etc. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Sequence + +TERMINAL_SCHEMA = "loop-engineer/terminal@1" + + +@dataclass(frozen=True) +class EngineOutcome: + """Engine-agnostic description of how a host run ended.""" + + reached_end: bool + external_error: str | None = None + budget_exhausted: bool = False + human_abort: bool = False + artifacts: Sequence[str] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "artifacts", tuple(str(a) for a in self.artifacts)) + + +def _valid_checks(checks: object) -> bool: + return ( + isinstance(checks, list) + and bool(checks) + and all(isinstance(c, dict) and "id" in c and isinstance(c.get("passed"), bool) for c in checks) + ) + + +def _valid_gate(gate: dict) -> bool: + """Structurally a ``holdout_gate.decide`` result: verdict + flag + the + per-check ``visible``/``holdout`` evidence arrays. A hand-typed stub with + no check evidence is not a gate run.""" + return ( + isinstance(gate.get("verdict"), str) + and isinstance(gate.get("false_completion"), bool) + and _valid_checks(gate.get("visible")) + and _valid_checks(gate.get("holdout")) + ) + + +def _valid_anticheat(anticheat: dict) -> bool: + return isinstance(anticheat.get("findings"), list) and "downgrade_to" in anticheat + + +def to_terminal_state( + outcome: EngineOutcome, + gate_verdict: dict | None, + anticheat: dict | None, + criteria_met: dict[str, bool | None], +) -> dict: + """Project an engine terminal + gate/anticheat evidence into a terminal@1 body. + + ``criteria_met`` maps each SPEC criterion id to the pass/fail of its mapped + check; ``None`` means the criterion has no mapped check at all -> + ``FailedSpecGap``. In the returned body ``None`` coerces to ``False`` + (unproven is not met). + """ + gate = gate_verdict if isinstance(gate_verdict, dict) else {} + ac = anticheat if isinstance(anticheat, dict) else {} + false_completion = gate.get("false_completion") is True + + def body(state: str, reason: str) -> dict: + return { + "schema": TERMINAL_SCHEMA, + "state": state, + "criteria_met": {str(k): v is True for k, v in criteria_met.items()}, + "evidence": list(outcome.artifacts), + "false_completion": false_completion, + "reason": reason, + } + + if _valid_anticheat(ac) and ac.get("downgrade_to") == "FailedSafety": + return body("FailedSafety", "anticheat: critical gate-tampering finding") + if outcome.human_abort: + return body("AbortedByHuman", "operator interrupt / human abort signal") + if outcome.external_error: + return body("FailedBlocked", f"unrecoverable external block: {outcome.external_error}") + if outcome.budget_exhausted: + return body("FailedBudget", "engine budget cap hit (steps/tokens/wall-clock/cost)") + unmapped = sorted(str(k) for k, v in criteria_met.items() if v is None) + if unmapped: + return body("FailedSpecGap", "criteria with no mapped check: " + ", ".join(unmapped)) + if not _valid_anticheat(ac): + return body("FailedUnverifiable", "no anticheat result — cannot certify (fail closed)") + if not _valid_gate(gate): + return body("FailedUnverifiable", "no holdout gate result — cannot certify (fail closed)") + if ac.get("downgrade_to") == "FailedUnverifiable": + return body("FailedUnverifiable", "anticheat: high-severity finding") + if gate["verdict"] != "Succeeded": + if false_completion: + return body("FailedUnverifiable", "visible passed but holdout failed — false completion") + return body("FailedUnverifiable", f"gate verdict {gate['verdict']!r} — cannot certify Succeeded") + if false_completion: + return body("FailedUnverifiable", "gate flags false_completion — refusing Succeeded") + if not any(v is True for v in criteria_met.values()): + return body("FailedUnverifiable", "green gate but no met criterion — cannot certify") + if not outcome.artifacts: + return body("FailedUnverifiable", "green gate but no evidence artifacts — cannot certify") + if not outcome.reached_end: + return body("FailedUnverifiable", "engine did not reach its own terminal signal") + return body("Succeeded", "holdout gate green, anticheat clean, criteria met with evidence") +``` + +Note: remove the unused `field` import if the linter flags it (keep imports minimal: `dataclass` only). + +- [ ] **Step 1.4: Run tests to verify they pass** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_integrations.py` +Expected: all PASS (≈14 tests) + +- [ ] **Step 1.5: Run the full suite + compile gate (regression)** + +Run: `uv run --with pytest --with pyyaml --with jsonschema python -B -m pytest -q -p no:cacheprovider scripts && python3 -B -m py_compile loop/*.py scripts/*.py` +Expected: baseline + new tests pass, no regressions + +- [ ] **Step 1.6: Commit** + +```bash +git add loop/integrations.py scripts/test_integrations.py +git commit -m "feat(st3): loop/integrations.py — EngineOutcome -> typed-terminal projection" +``` + +--- + +### Task 2: LangGraph recipe upgraded in place to the ST3 bar + +**Files:** +- Modify: `examples/langgraph-emit/graph_example.py` (full rewrite below) +- Modify: `examples/langgraph-emit/README.md` +- Modify: `docs/integrations/langgraph.md` (full rewrite below) +- Create: `scripts/test_langgraph_recipe_st3.py` +- Modify: `.github/workflows/ci.yml` (recipe-langgraph job runs both test files) +- Do NOT touch: `scripts/test_langgraph_recipe.py` (must stay byte-identical and green) + +**Interfaces:** +- Consumes: `loop.integrations.EngineOutcome` / `to_terminal_state` (Task 1), `loop.emit.*`, `holdout_gate.decide`, `anticheat_scan.scan` (imported via `loop._resources.tools_dir()`). +- Produces: `python graph_example.py [--sabotage-holdout]` — happy path emits a `Succeeded` contract that passes `doctor` AND `metrics` cleanly; sabotage path emits `FailedUnverifiable` with `false_completion: true`. + +- [ ] **Step 2.1: Verify the current LangGraph API via Context7** + +Call `resolve-library-id` for "LangGraph", then `query-docs` for "StateGraph add_node add_edge START END compile invoke current API". Confirm the shipped pattern (`StateGraph(State).add_node(fn).add_edge(START, "do_work")...compile().invoke({...})`) is still current (it runs green in CI today, so expect confirmation). Note the version CI resolves for the doc's "verified against" line (see Step 2.6). + +- [ ] **Step 2.2: Write the failing test** + +Create `scripts/test_langgraph_recipe_st3.py`: + +```python +"""ST3 acceptance for the LangGraph recipe: the certify node routes through +EngineOutcome + to_terminal_state (gate + anticheat wired), the emitted +contract passes doctor, `loop metrics` scores the run clean (closes the +FCR-1.0 follow-up), and the false-completion invariant holds under sabotage. +Env-guarded: langgraph is a dev dependency of the example only.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("langgraph") + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXAMPLE = REPO_ROOT / "examples" / "langgraph-emit" / "graph_example.py" + + +def _run_example(workspace: Path, *args: str) -> subprocess.CompletedProcess: + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) + return subprocess.run( + [sys.executable, "-B", str(EXAMPLE), str(workspace), *args], + cwd=workspace.parent, env=env, capture_output=True, text=True, + ) + + +def _cli(cmd: str, workspace: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", cmd, str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + + +def test_happy_path_is_doctor_clean_and_metrics_clean(tmp_path): + ws = tmp_path / "graph-run" + proc = _run_example(ws) + assert proc.returncode == 0, proc.stdout + proc.stderr + + doctored = _cli("doctor", ws) + assert doctored.returncode == 0, doctored.stdout + assert json.loads(doctored.stdout)["ok"] is True + + terminal = json.loads((ws / ".loop" / "terminal_state.json").read_text()) + assert terminal["state"] == "Succeeded" + assert terminal["false_completion"] is False + assert terminal["evidence"] + + metrics = _cli("metrics", ws) + assert metrics.returncode == 0, metrics.stdout + metrics.stderr + card = json.loads(metrics.stdout) + assert card["false_completion_rate"] == 0.0 + assert card["false_completions"] == 0 + assert card["iterations_claiming_success"] >= 1 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["unmatched_verify"] == [] + assert prov["unrecognized_outcomes"] == [] + assert prov["fcr_methods_agree"] is True + + +def test_sabotage_holdout_is_false_completion_never_succeeded(tmp_path): + ws = tmp_path / "graph-run-sabotaged" + proc = _run_example(ws, "--sabotage-holdout") + # the recipe exits non-zero on a non-Succeeded terminal, but still emits it + terminal = json.loads((ws / ".loop" / "terminal_state.json").read_text()) + assert terminal["state"] == "FailedUnverifiable" + assert terminal["state"] != "Succeeded" + assert terminal["false_completion"] is True, (terminal, proc.stdout, proc.stderr) + + doctored = _cli("doctor", ws) + assert json.loads(doctored.stdout)["ok"] is True # an honest failure is a valid contract +``` + +- [ ] **Step 2.3: Run it to make sure it fails** + +Run: `uv run --with pytest --with pyyaml --with langgraph python -B -m pytest -q -p no:cacheprovider scripts/test_langgraph_recipe_st3.py` +Expected: FAIL (no `--sabotage-holdout` support; no metrics-clean artifacts yet) + +- [ ] **Step 2.4: Rewrite `examples/langgraph-emit/graph_example.py`** + +```python +"""A LangGraph graph whose END is reachable only through a certify node. + +The certify node runs the SAME split the loop optimized against — a visible +check plus a WITHHELD holdout check — through the real holdout gate and +anticheat scan, projects the graph's terminal through loop.integrations, and +records the result via loop.emit (which refuses a dishonest Succeeded). + + python graph_example.py [--sabotage-holdout] + +--sabotage-holdout makes do_work write output that passes the visible check +but fails the holdout — the measurable false-completion event: the terminal +becomes FailedUnverifiable with false_completion: true, never Succeeded. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TypedDict + +from langgraph.graph import END, START, StateGraph + +from loop import emit +from loop._resources import tools_dir +from loop.integrations import EngineOutcome, to_terminal_state + +sys.path.insert(0, str(tools_dir())) +import anticheat_scan # noqa: E402 +import holdout_gate # noqa: E402 + +EXPECTED = "hello from langgraph\n" + + +class State(TypedDict): + workspace: str + sabotage: bool + terminal_state: str + + +def do_work(state: State) -> dict: + out = Path(state["workspace"]) / "artifact.txt" + out.write_text("HELLO stub\n" if state["sabotage"] else EXPECTED, encoding="utf-8") + return {} + + +def certify(state: State) -> dict: + ws = Path(state["workspace"]) + artifact = ws / "artifact.txt" + + # 1. The gate: visible = what the loop optimized against; holdout = withheld. + visible = [{"id": "artifact-exists", "passed": artifact.is_file()}] + holdout = [{ + "id": "artifact-content", + "passed": artifact.is_file() and artifact.read_text(encoding="utf-8") == EXPECTED, + }] + gate = holdout_gate.decide(visible, holdout) + ac = anticheat_scan.scan(diff_text="", trajectory=[str(artifact)]) + + # 2. Evidence artifacts: the gate verdict + a verify bundle metrics can join. + art_dir = ws / ".loop" / "artifacts" + art_dir.mkdir(parents=True, exist_ok=True) + (art_dir / "holdout-verdict.json").write_text(json.dumps(gate, indent=2) + "\n", encoding="utf-8") + bundle = { + "task": "T1", + "verify": "certify node — holdout_gate.decide over visible+holdout", + "outcome": "PASS" if gate["verdict"] == "Succeeded" else "FAIL", + "iteration_id": 1, + "criteria": {"1": gate["verdict"] == "Succeeded"}, + } + (art_dir / "verify-T1.json").write_text(json.dumps(bundle, indent=2) + "\n", encoding="utf-8") + + # 3. Project the graph terminal into a typed state; write via emit only. + terminal = to_terminal_state( + outcome=EngineOutcome( + reached_end=True, + artifacts=[".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + ), + gate_verdict=gate, + anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + passed = terminal["state"] == "Succeeded" + emit.append_iteration( + ws, iteration_id=1, outcome="task_passed" if passed else "task_failed", + task_id="T1", + actions=["wrote artifact.txt", "ran holdout_gate.decide + anticheat_scan.scan"], + verify_cmd="holdout_gate.decide(visible, holdout)", verify_outcome=gate["verdict"], + notes="verify bundle: verify-T1.json; gate verdict: holdout-verdict.json", + ) + emit.append_receipt(ws, iteration_id=1, role="orchestrate", model="deterministic-demo", outcome="ok") + emit.terminate( + ws, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return {"terminal_state": terminal["state"]} + + +def main(workspace: str, sabotage: bool) -> int: + emit.open_contract(workspace) + graph = ( + StateGraph(State) + .add_node(do_work) + .add_node(certify) + .add_edge(START, "do_work") + .add_edge("do_work", "certify") + .add_edge("certify", END) # certify IS the only path to END + .compile() + ) + result = graph.invoke({"workspace": workspace, "sabotage": sabotage, "terminal_state": ""}) + print(f"terminal: {result['terminal_state']} — validate: python3 -m loop doctor {workspace}") + return 0 if result["terminal_state"] == "Succeeded" else 1 + + +if __name__ == "__main__": + args = sys.argv[1:] + sabotage = "--sabotage-holdout" in args + targets = [a for a in args if not a.startswith("--")] + if len(targets) != 1: + print("usage: python graph_example.py [--sabotage-holdout]", file=sys.stderr) + raise SystemExit(2) + raise SystemExit(main(targets[0], sabotage)) +``` + +Compatibility check for the untouched `scripts/test_langgraph_recipe.py`: it runs the happy path and asserts exit 0, doctor ok, `state == "Succeeded"`, non-empty evidence — all still true. + +- [ ] **Step 2.5: Run both LangGraph test files** + +Run: `uv run --with pytest --with pyyaml --with langgraph python -B -m pytest -q -p no:cacheprovider scripts/test_langgraph_recipe.py scripts/test_langgraph_recipe_st3.py` +Expected: all PASS. If `metrics` reports `evidence_backed False` or unmatched claims, debug against the metrics contract (bundle needs `iteration_id`; verdict artifact must be the verbatim `decide()` output). + +- [ ] **Step 2.6: Rewrite `docs/integrations/langgraph.md`** + +Content requirements (write it, keeping ≤90 lines): +1. Title: `# LangGraph — gate the graph, then emit proof-of-done`. +2. One-paragraph complement framing: LangGraph owns the ORCHESTRATE tier and stays the runtime; Loop Engineer adds the contract/proof tier above it. Never "replaces". +3. The pattern: a `certify` node is the only edge into `END`; it runs `holdout_gate.decide` + `anticheat_scan.scan`, projects through `to_terminal_state`, writes via `emit`. Include the certify-node snippet (condensed from Step 2.4's `certify`). +4. Mapping table specialized to LangGraph: + +| LangGraph signal | Typed terminal state | +|---|---| +| graph reached `END`, holdout green + anticheat clean | `Succeeded` | +| graph reached `END`, visible green / holdout red | `FailedUnverifiable` (`false_completion: true`) | +| `GraphRecursionError` (LangGraph's own step cap) | `FailedBudget` | +| caught tool/credential exception | `FailedBlocked` | +| operator interrupt | `AbortedByHuman` | + +5. The zero-install copy-paste mode — inline this ~15-line equivalent and say the installable module is convenience, not a requirement: + +```python +def to_terminal(gate, anticheat, criteria_met, evidence, + *, human_abort=False, blocked=None, over_budget=False): + fc = gate.get("false_completion") is True + if anticheat.get("downgrade_to") == "FailedSafety": state = "FailedSafety" + elif human_abort: state = "AbortedByHuman" + elif blocked: state = "FailedBlocked" + elif over_budget: state = "FailedBudget" + elif any(v is None for v in criteria_met.values()): state = "FailedSpecGap" + elif (not gate or not anticheat or anticheat.get("downgrade_to") + or gate.get("verdict") != "Succeeded" or fc + or not any(criteria_met.values()) or not evidence): state = "FailedUnverifiable" + else: state = "Succeeded" + return {"schema": "loop-engineer/terminal@1", "state": state, + "criteria_met": {k: v is True for k, v in criteria_met.items()}, + "evidence": list(evidence), "false_completion": fc} +``` + +6. CI gating block (`pip install loop-engineer` + `loop doctor run/` + `loop metrics run/`). +7. A "Verified against `langgraph` vX.Y.Z (YYYY-MM-DD)" line — fill X.Y.Z from `python -m pip show langgraph` in the environment where Step 2.5 ran. +8. Link to `examples/langgraph-emit/`. + +Also update `examples/langgraph-emit/README.md`: describe the certify node, the `--sabotage-holdout` false-completion demo, the `loop metrics` clean score, and keep the run instructions (`pip install loop-engineer langgraph`, run from a repo checkout for the `scripts/` gate tools, or rely on the wheel's bundled tools via `loop._resources`). + +- [ ] **Step 2.7: Update the CI job** + +In `.github/workflows/ci.yml`, change the recipe-langgraph test step to: + +```yaml + - name: LangGraph recipe end-to-end + run: python -B -m pytest -q -p no:cacheprovider scripts/test_langgraph_recipe.py scripts/test_langgraph_recipe_st3.py +``` + +- [ ] **Step 2.8: Full local gates** + +Run: `uv run --with pytest --with pyyaml --with jsonschema --with langgraph python -B -m pytest -q -p no:cacheprovider scripts && python3 -B scripts/self_eval.py && python3 -B scripts/validate_frontmatter.py` +Expected: green + +- [ ] **Step 2.9: Commit** + +```bash +git add examples/langgraph-emit/ docs/integrations/langgraph.md scripts/test_langgraph_recipe_st3.py .github/workflows/ci.yml +git commit -m "feat(st3): upgrade LangGraph recipe to the gate+adapter bar — metrics-clean, invariant-pinned" +``` + +--- + +### Task 3: Temporal recipe (new) + +**Files:** +- Create: `examples/temporal-certify/workflow_example.py` +- Create: `examples/temporal-certify/README.md` +- Create: `docs/integrations/temporal.md` +- Create: `scripts/test_temporal_recipe.py` +- Modify: `.github/workflows/ci.yml` (new `recipe-temporal` job) + +**Interfaces:** +- Consumes: Task 1's adapter, `loop.emit`, gate tools via `tools_dir()`. +- Produces: `run_and_certify(client, workspace, *, sabotage=False, wf_id)` (async), `certify_workflow_failure(workspace, cause) -> dict` (sync; maps a `WorkflowFailureError.cause` through `map_workflow_failure` and terminates), `map_workflow_failure(cause) -> EngineOutcome` (pure), `CertifiedGoalWorkflow`, `do_work_activity`, `certify_activity`, `TASK_QUEUE`. + +- [ ] **Step 3.1: Verify the Temporal Python API via Context7** + +Already verified for this plan (2026-07-08, `/temporalio/sdk-python`): `Client.connect`, `Worker(client, task_queue=..., workflows=[...], activities=[...])` as async context manager, `WorkflowEnvironment.start_local()`, `@activity.defn` / `@workflow.defn` / `@workflow.run`, `workflow.execute_activity(fn, arg, start_to_close_timeout=..., retry_policy=RetryPolicy(maximum_attempts=N))`, `handle.cancel()` then `await handle.result()` raises `WorkflowFailureError` with `CancelledError` as cause. At build time, re-query for: (a) exception classes `temporalio.exceptions.{CancelledError, TimeoutError, ActivityError}` and what `run_timeout` expiry surfaces as; (b) whether module-level non-deterministic imports in the workflow file need `with workflow.unsafe.imports_passed_through():` (they do by default) or `@workflow.defn(sandboxed=False)`. Record the installed version for the doc note. + +- [ ] **Step 3.2: Write the failing test** + +Create `scripts/test_temporal_recipe.py`: + +```python +"""ST3 acceptance for the Temporal recipe: a certify ACTIVITY is the workflow's +only path to a returned result; the emitted contract passes doctor; the +false-completion invariant holds under sabotage; cancellation maps to +AbortedByHuman. Env-guarded: temporalio is a dev dependency of the example +only — the package stays zero-dependency. Uses asyncio.run (no pytest-asyncio). +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import subprocess +import sys +import uuid +from datetime import timedelta +from pathlib import Path + +import pytest + +pytest.importorskip("temporalio") + +from temporalio.client import WorkflowFailureError # noqa: E402 +from temporalio.testing import WorkflowEnvironment # noqa: E402 +from temporalio.worker import Worker # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +_EXAMPLE_PATH = REPO_ROOT / "examples" / "temporal-certify" / "workflow_example.py" +_spec = importlib.util.spec_from_file_location("workflow_example", _EXAMPLE_PATH) +recipe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(recipe) + + +def _doctor(workspace: Path) -> dict: + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "doctor", str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + return json.loads(proc.stdout) + + +def _terminal(workspace: Path) -> dict: + return json.loads((workspace / ".loop" / "terminal_state.json").read_text()) + + +def test_recipe_end_to_end(tmp_path): + async def scenario(): + async with await WorkflowEnvironment.start_local() as env: + async with Worker( + env.client, + task_queue=recipe.TASK_QUEUE, + workflows=[recipe.CertifiedGoalWorkflow], + activities=[recipe.do_work_activity, recipe.certify_activity], + ): + await recipe.run_and_certify( + env.client, str(tmp_path / "happy"), sabotage=False, + wf_id=f"happy-{uuid.uuid4()}", + ) + await recipe.run_and_certify( + env.client, str(tmp_path / "sabotaged"), sabotage=True, + wf_id=f"sab-{uuid.uuid4()}", + ) + # Cancellation -> AbortedByHuman (mapped host-side) + ws_cancel = tmp_path / "cancelled" + recipe.emit.open_contract(ws_cancel) + handle = await env.client.start_workflow( + recipe.CertifiedGoalWorkflow.run, + recipe.WorkArgs(workspace=str(ws_cancel), sabotage=False, hold_seconds=30), + id=f"cancel-{uuid.uuid4()}", task_queue=recipe.TASK_QUEUE, + ) + await handle.cancel() + try: + await handle.result() + except WorkflowFailureError as exc: + recipe.certify_workflow_failure(str(ws_cancel), exc.cause) + else: # pragma: no cover - cancellation must surface + raise AssertionError("expected WorkflowFailureError after cancel") + + asyncio.run(scenario()) + + happy = _terminal(tmp_path / "happy") + assert happy["state"] == "Succeeded" + assert happy["false_completion"] is False + assert happy["evidence"] + assert _doctor(tmp_path / "happy")["ok"] is True + + sab = _terminal(tmp_path / "sabotaged") + assert sab["state"] == "FailedUnverifiable" + assert sab["state"] != "Succeeded" + assert sab["false_completion"] is True + assert _doctor(tmp_path / "sabotaged")["ok"] is True + + cancelled = _terminal(tmp_path / "cancelled") + assert cancelled["state"] == "AbortedByHuman" + assert _doctor(tmp_path / "cancelled")["ok"] is True + + +def test_map_workflow_failure_covers_blocked_and_budget(): + from temporalio.exceptions import ActivityError, TimeoutError as TemporalTimeoutError + + blocked = recipe.map_workflow_failure( + ActivityError("activity failed", *_activity_error_extra_args()) + if False else _make(ActivityError, "activity failed") + ) + assert blocked.external_error + budget = recipe.map_workflow_failure(_make(TemporalTimeoutError, "workflow timeout")) + assert budget.budget_exhausted is True + + +def _make(exc_type, message): + """Best-effort construction of a temporalio failure for the pure mapper. + If a class needs richer args in the installed version, fall back to a + minimal subclass instance carrying only the type identity.""" + try: + return exc_type(message) + except TypeError: + stub = type(exc_type.__name__, (exc_type,), {"__init__": lambda self: None}) + return stub() + + +def _activity_error_extra_args(): # placeholder for versions needing more args + return () +``` + +Note to implementer: `test_map_workflow_failure_covers_blocked_and_budget` intentionally uses `_make` because temporalio failure constructors vary by version — verify the installed signature at build time (Step 3.1) and simplify this test to direct construction if the version allows; the mapper itself must dispatch on `isinstance` only. + +- [ ] **Step 3.3: Run it to make sure it fails** + +Run: `uv run --with pytest --with pyyaml --with temporalio python -B -m pytest -q -p no:cacheprovider scripts/test_temporal_recipe.py` +Expected: FAIL/ERROR — `examples/temporal-certify/workflow_example.py` does not exist. (First run downloads the Temporal dev-server binary; allow ~a minute.) + +- [ ] **Step 3.4: Implement `examples/temporal-certify/workflow_example.py`** + +```python +"""A Temporal workflow whose only path to a returned result is a certify +ACTIVITY — activities do I/O, the workflow stays deterministic. + +Temporal owns durability (the run survives crashes); Loop Engineer owns the +on-disk success/evidence truth. The certify activity runs the visible/holdout +split through the real holdout gate + anticheat scan, projects through +loop.integrations, and records the result via loop.emit. Host-side failure +mapping: cancellation -> AbortedByHuman, retry-policy exhaustion on an +external dependency -> FailedBlocked, workflow timeout -> FailedBudget. + + python workflow_example.py [--sabotage-holdout] + +(standalone mode starts a local Temporal dev server via +temporalio.testing.WorkflowEnvironment.start_local — first run downloads it) +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path + +from temporalio import activity, workflow +from temporalio.client import Client, WorkflowFailureError +from temporalio.common import RetryPolicy +from temporalio.exceptions import ActivityError, CancelledError +from temporalio.exceptions import TimeoutError as TemporalTimeoutError + +with workflow.unsafe.imports_passed_through(): + from loop import emit + from loop._resources import tools_dir + from loop.integrations import EngineOutcome, to_terminal_state + +sys.path.insert(0, str(tools_dir())) +with workflow.unsafe.imports_passed_through(): + import anticheat_scan # noqa: E402 + import holdout_gate # noqa: E402 + +EXPECTED = "hello from temporal\n" +TASK_QUEUE = "loop-engineer-certify-demo" + + +@dataclass +class WorkArgs: + workspace: str + sabotage: bool = False + hold_seconds: int = 0 + + +@activity.defn +async def do_work_activity(args: WorkArgs) -> str: + out = Path(args.workspace) / "artifact.txt" + out.write_text("HELLO stub\n" if args.sabotage else EXPECTED, encoding="utf-8") + return str(out) + + +@activity.defn +async def certify_activity(args: WorkArgs) -> dict: + ws = Path(args.workspace) + artifact = ws / "artifact.txt" + visible = [{"id": "artifact-exists", "passed": artifact.is_file()}] + holdout = [{ + "id": "artifact-content", + "passed": artifact.is_file() and artifact.read_text(encoding="utf-8") == EXPECTED, + }] + gate = holdout_gate.decide(visible, holdout) + ac = anticheat_scan.scan(diff_text="", trajectory=[str(artifact)]) + + art_dir = ws / ".loop" / "artifacts" + art_dir.mkdir(parents=True, exist_ok=True) + (art_dir / "holdout-verdict.json").write_text(json.dumps(gate, indent=2) + "\n", encoding="utf-8") + + terminal = to_terminal_state( + outcome=EngineOutcome(reached_end=True, artifacts=[".loop/artifacts/holdout-verdict.json"]), + gate_verdict=gate, anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + passed = terminal["state"] == "Succeeded" + emit.append_iteration( + str(ws), iteration_id=1, outcome="task_passed" if passed else "task_failed", + task_id="T1", actions=["do_work_activity wrote artifact.txt", "certify_activity gated it"], + verify_cmd="holdout_gate.decide(visible, holdout)", verify_outcome=gate["verdict"], + ) + emit.append_receipt(str(ws), iteration_id=1, role="orchestrate", model="deterministic-demo", outcome="ok") + emit.terminate( + str(ws), state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return {"state": terminal["state"], "false_completion": terminal["false_completion"]} + + +@workflow.defn +class CertifiedGoalWorkflow: + @workflow.run + async def run(self, args: WorkArgs) -> dict: + if args.hold_seconds: + await asyncio.sleep(args.hold_seconds) # durable timer (cancel/timeout demos) + await workflow.execute_activity( + do_work_activity, args, + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy(maximum_attempts=2), + ) + return await workflow.execute_activity( + certify_activity, args, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=RetryPolicy(maximum_attempts=1), + ) + + +def map_workflow_failure(cause: BaseException | None) -> EngineOutcome: + """Pure projection of a WorkflowFailureError cause onto EngineOutcome.""" + if isinstance(cause, CancelledError): + return EngineOutcome(reached_end=False, human_abort=True) + if isinstance(cause, TemporalTimeoutError): + return EngineOutcome(reached_end=False, budget_exhausted=True) + if isinstance(cause, ActivityError): + return EngineOutcome(reached_end=False, external_error=f"activity retries exhausted: {cause}") + return EngineOutcome(reached_end=False, external_error=str(cause) or "unknown engine failure") + + +def certify_workflow_failure(workspace: str, cause: BaseException | None) -> dict: + """Terminate an already-opened contract from a workflow failure, honestly.""" + terminal = to_terminal_state( + outcome=map_workflow_failure(cause), gate_verdict={}, anticheat={}, + criteria_met={"1": False}, + ) + emit.terminate( + workspace, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return terminal + + +async def run_and_certify(client: Client, workspace: str, *, sabotage: bool, wf_id: str) -> dict: + emit.open_contract(workspace) + try: + return await client.execute_workflow( + CertifiedGoalWorkflow.run, WorkArgs(workspace=workspace, sabotage=sabotage), + id=wf_id, task_queue=TASK_QUEUE, + ) + except WorkflowFailureError as exc: + return certify_workflow_failure(workspace, exc.cause) + + +async def _amain(workspace: str, sabotage: bool) -> int: + from temporalio.testing import WorkflowEnvironment + from temporalio.worker import Worker + + async with await WorkflowEnvironment.start_local() as env: + async with Worker( + env.client, task_queue=TASK_QUEUE, + workflows=[CertifiedGoalWorkflow], + activities=[do_work_activity, certify_activity], + ): + result = await run_and_certify(env.client, workspace, sabotage=sabotage, wf_id="demo-run") + print(f"terminal: {result['state']} — validate: python3 -m loop doctor {workspace}") + return 0 if result["state"] == "Succeeded" else 1 + + +if __name__ == "__main__": + args = sys.argv[1:] + sabotage = "--sabotage-holdout" in args + targets = [a for a in args if not a.startswith("--")] + if len(targets) != 1: + print("usage: python workflow_example.py [--sabotage-holdout]", file=sys.stderr) + raise SystemExit(2) + raise SystemExit(asyncio.run(_amain(targets[0], sabotage))) +``` + +Build-time adjustments the implementer owns (from Step 3.1 verification): the exact sandbox-import pattern (`imports_passed_through` placement vs `@workflow.defn(sandboxed=False)` — prefer the former; if the sandbox still rejects the module because activities and gate tools share the file, fall back to `sandboxed=False` with a one-line comment naming why), and whether `certify_activity`'s result dict serializes as-is (it does — plain JSON). + +- [ ] **Step 3.5: Run the tests until green** + +Run: `uv run --with pytest --with pyyaml --with temporalio python -B -m pytest -q -p no:cacheprovider scripts/test_temporal_recipe.py` +Expected: PASS (2 tests). Iterate on sandbox/exception-class details per Step 3.1 notes, keeping the mapper `isinstance`-only. + +- [ ] **Step 3.6: Write `docs/integrations/temporal.md` and the example README** + +`docs/integrations/temporal.md` (≤90 lines) mirrors the LangGraph doc structure: +1. Title: `# Temporal — durable execution below, proof-of-done above`. +2. Complement framing: Temporal owns the EXECUTE tier — the run *survives crashes*; it says nothing about whether the work is *correct*. Loop Engineer adds the correctness/termination contract on top. Never "replaces". +3. The pattern: a `certify` activity is the workflow's only path to a returned result (activities do I/O; the workflow stays deterministic). Condensed `certify_activity` + `CertifiedGoalWorkflow` snippet. +4. Mapping table specialized to Temporal: + +| Temporal signal | Typed terminal state | +|---|---| +| workflow returned via certify activity, holdout green + anticheat clean | `Succeeded` | +| workflow returned, visible green / holdout red | `FailedUnverifiable` (`false_completion: true`) | +| workflow `CancelledError` | `AbortedByHuman` | +| activity `RetryPolicy` exhaustion on an external dependency | `FailedBlocked` | +| workflow timeout (`run_timeout`) | `FailedBudget` | + +5. The same ~15-line zero-install copy-paste block as the LangGraph doc (identical code — the adapter is engine-neutral; say so). +6. CI gating block (`loop doctor run/`). +7. "Verified against `temporalio` vX.Y.Z (YYYY-MM-DD)" — from `pip show temporalio` where Step 3.5 ran. +8. Link to `examples/temporal-certify/`. + +`examples/temporal-certify/README.md`: what it shows (certify-activity pattern, sabotage demo, host-side failure mapping incl. cancel→AbortedByHuman), how to run (`pip install loop-engineer temporalio`, `python workflow_example.py demo-run/` — notes the first run downloads the local dev server), and the doctor command. + +- [ ] **Step 3.7: Add the CI job** + +Append to `.github/workflows/ci.yml`: + +```yaml + recipe-temporal: + name: recipe (temporal) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Cache Temporal dev server + uses: actions/cache@v4 + with: + path: ~/.temporalio + key: temporal-dev-server-${{ runner.os }} + - name: Install recipe dependencies + run: python -m pip install --upgrade pip pyyaml pytest jsonschema temporalio + - name: Temporal recipe end-to-end + run: python -B -m pytest -q -p no:cacheprovider scripts/test_temporal_recipe.py +``` + +(Verify the dev-server cache path at build time — if `start_local` caches elsewhere or not at all, keep the cache step keyed to the observed path or drop it; the job must pass either way. If the job proves flaky in CI, the e2e stays env-guarded and the spec's containment note applies.) + +- [ ] **Step 3.8: Full local gates** + +Run: `uv run --with pytest --with pyyaml --with jsonschema --with langgraph --with temporalio python -B -m pytest -q -p no:cacheprovider scripts && python3 -B -m py_compile loop/*.py scripts/*.py examples/temporal-certify/workflow_example.py` +Expected: green + +- [ ] **Step 3.9: Commit** + +```bash +git add examples/temporal-certify/ docs/integrations/temporal.md scripts/test_temporal_recipe.py .github/workflows/ci.yml +git commit -m "feat(st3): Temporal recipe — certify activity gates the workflow result" +``` + +--- + +### Task 4: PR-A assembly — CHANGELOG, gates, ship + +**Files:** +- Modify: `CHANGELOG.md` (add `## Unreleased` section at top, below `## Errata`) + +- [ ] **Step 4.1: CHANGELOG entry** + +Add under a new `## Unreleased` heading (above `## 0.7.0`): + +```markdown +## Unreleased + +**ST3 — integration adapters.** `loop/integrations.py`: an engine-neutral, +pure-stdlib projection (`EngineOutcome` + `to_terminal_state`) from any +engine's "the run ended" signal onto the 7 typed terminal states, with the +fixed precedence safety → human → blocked → budget → spec-gap → gate verdict. +`Succeeded` is reachable only through a green `holdout_gate.decide` verdict, +a clean anticheat sweep, a met criterion, and evidence; `false_completion` is +copied from the gate, never synthesized; missing gate/anticheat input fails +closed to `FailedUnverifiable`. The LangGraph recipe is upgraded in place to +this bar (its run now scores clean under `loop metrics` — closing the +recorded FCR-1.0 follow-up) and a Temporal recipe lands +(`examples/temporal-certify/`, certify-activity pattern, cancellation → +`AbortedByHuman`, retry exhaustion → `FailedBlocked`, timeout → +`FailedBudget`). Both recipes pin the false-completion invariant +(visible-green/holdout-red → `FailedUnverifiable` with +`false_completion: true`, never `Succeeded`) and pass the doctor round-trip. +``` + +Check no line matches the anticheat comment shapes (`# expected|hardcode|hack|cheat|to pass`) — `test_docs_version.py::test_changelog_entry_has_no_anticheat_comment_shapes` gates this. + +- [ ] **Step 4.2: Full gate sweep** + +```bash +python3 -B scripts/validate_frontmatter.py +python3 -B scripts/self_eval.py +uv run --with pytest --with pyyaml --with jsonschema --with langgraph --with temporalio python -B -m pytest -q -p no:cacheprovider scripts +python3 -B -m py_compile loop/*.py scripts/*.py +python3 -B -m loop doctor examples/coverage-repair +python3 -B -m loop inspect examples/coverage-repair +``` +Expected: all green; pytest ≥ baseline (372/10) plus the new tests. + +- [ ] **Step 4.3: Commit, push, open PR-A** + +```bash +git add CHANGELOG.md +git commit -m "docs(changelog): ST3 unreleased entry" +git push -u origin feat/v0.8.0-composes-the-field +gh pr create --title "feat(st3): integration adapters — loop/integrations.py + LangGraph & Temporal recipes" --body-file /tmp/claude-pr-a-body.md +``` + +Write the PR body file first: summary (spec link, the three deliverables, acceptance-criteria checklist §5 items 1–3 + 6), test plan (gate sweep above + the two recipe CI jobs). End the body with the standard attribution footer only if repo convention requires it (this repo's history: no attribution — keep it clean, per global settings). + +- [ ] **Step 4.4: CI green, then merge** + +Watch `gh pr checks --watch`. All jobs green (gates ×3 pythons, recipe-langgraph, recipe-temporal, action-dogfood) → squash-merge (`gh pr merge --squash`). If a recipe job fails on engine-version drift, fix in-place (the "verified against vX" note updates too). + +--- + +## PR-B — ST4: contributor funnel + release cut + +Branch off fresh main after PR-A merges: `git checkout main && git pull && git checkout -b feat/v0.8.0-st4-funnel`. + +### Task 5: Foreign-harness inspect adapter (read-only) + vendored fixture + +**Files:** +- Create: `loop/foreign.py` +- Modify: `scripts/inspect_loop.py` (seam + labels ONLY, per Global Constraints) +- Create: `examples/superpowers-run/README.md` +- Create: `examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md` +- Create: `examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md` +- Create: `examples/superpowers-run/.superpowers/sdd/progress.md` +- Test: `scripts/test_foreign_inspect.py` + +**Interfaces:** +- Produces: `loop.foreign.detect_foreign_layout(target) -> str | None` (returns `"superpowers"` or `None`; a native `.loop/state.json` always wins) and `loop.foreign.map_foreign_paths(target) -> LoopPaths | None`. +- Consumes: `loop.paths.LoopPaths` (frozen dataclass, 10 fields — construct directly). +- `inspect_loop.inspect_loop()` report gains, only when mapped: `"foreign_layout": "superpowers"`, `"advisory": true`. + +- [ ] **Step 5.1: Write the failing test** + +Create `scripts/test_foreign_inspect.py`: + +```python +"""ST4: the foreign-harness inspect adapter is a LAYOUT MAPPER, not a scorer +change. It maps a Superpowers-style run dir onto the LoopPaths surface the +inspector already consumes; the M2/M3-hardened scorer is not re-litigated. A +foreign harness with no holdout gate and no terminal record scores honestly +low — the regression tests pin that no run-recorded credit appears without an +on-disk gate, and that doctor does NOT get the mapping (inspect-only).""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from loop.foreign import detect_foreign_layout, map_foreign_paths # noqa: E402 + +FIXTURE = _REPO / "examples" / "superpowers-run" +NATIVE = _REPO / "examples" / "coverage-repair" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, Path(__file__).parent / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_detects_superpowers_layout_and_not_native_contracts(): + assert detect_foreign_layout(FIXTURE) == "superpowers" + assert detect_foreign_layout(NATIVE) is None # a native contract always wins + assert detect_foreign_layout(_REPO / "examples" / "naive-loop") is None + + +def test_mapping_points_at_superpowers_artifacts(): + paths = map_foreign_paths(FIXTURE) + assert paths is not None + assert paths.workspace == FIXTURE.resolve() + assert paths.spec.name == "2026-07-08-csv-dedupe-design.md" + assert paths.workflow.name == "2026-07-08-csv-dedupe.md" + assert paths.runlog.name == "progress.md" + assert map_foreign_paths(NATIVE) is None + + +def test_inspect_scores_fixture_low_and_labels_it_foreign(): + inspect_loop = _load("inspect_loop") + report = inspect_loop.inspect_loop(str(FIXTURE)) + assert report["foreign_layout"] == "superpowers" + assert report["advisory"] is True + assert report["verdict"] == "weak" + assert report["score"] < 50 + # honesty regression: NO run-recorded credit without an on-disk gate + assert not any("(invoked)" in p for p in report["present"]), report["present"] + assert report["terminal_states_covered"] < 7 + + +def test_native_reports_carry_no_foreign_label(): + inspect_loop = _load("inspect_loop") + report = inspect_loop.inspect_loop(str(NATIVE)) + assert "foreign_layout" not in report + assert "advisory" not in report + assert report["verdict"] == "strong" # flagship unchanged — scorer untouched + + +def test_cli_inspect_produces_scored_foreign_report(): + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "inspect", str(FIXTURE)], + cwd=_REPO, capture_output=True, text=True, + ) + report = json.loads(proc.stdout) + assert report["foreign_layout"] == "superpowers" + assert isinstance(report["score"], int) + + +def test_doctor_does_not_get_the_mapping(): + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "doctor", str(FIXTURE)], + cwd=_REPO, capture_output=True, text=True, + ) + assert proc.returncode != 0 # no .loop contract -> doctor honestly fails +``` + +- [ ] **Step 5.2: Run it to make sure it fails** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_foreign_inspect.py` +Expected: FAIL — `No module named 'loop.foreign'` + +- [ ] **Step 5.3: Implement `loop/foreign.py`** + +```python +"""Read-only foreign-harness layout mapping (ST4). + +Recognizes a run directory laid out by a foreign harness (currently: the +Superpowers spec/plan convention) and maps it onto the same ``LoopPaths`` +surface the inspector already consumes. A MAPPER, never a scorer: it points +the existing signals at foreign files and never manufactures credit — a +harness with no holdout gate and no terminal record scores honestly low, +which is the point. Synthesizing gate or verify artifacts here would be +dishonest and is out of scope by design. + +Used by ``inspect`` only; ``doctor`` stays unmapped (a foreign dir has no +contract to validate, and saying otherwise would be a false completion). +""" + +from __future__ import annotations + +from pathlib import Path + +from .paths import LoopPaths + +_SPECS_DIR = "docs/superpowers/specs" +_PLANS_DIR = "docs/superpowers/plans" +_JOURNALS = (".superpowers/sdd/progress.md", "docs/superpowers/journal.md") + + +def _newest_md(directory: Path) -> Path | None: + """Newest markdown file by name — Superpowers files are date-prefixed, so + lexicographic order is chronological order.""" + if not directory.is_dir(): + return None + files = sorted(p for p in directory.glob("*.md") if p.is_file()) + return files[-1] if files else None + + +def detect_foreign_layout(target: str | Path) -> str | None: + """Name the foreign layout of ``target``, or None. + + A native contract (``.loop/state.json``) always wins — foreign mapping + never shadows a real repo-OS contract. + """ + ws = Path(target) + if (ws / ".loop" / "state.json").is_file(): + return None + if _newest_md(ws / _SPECS_DIR) or _newest_md(ws / _PLANS_DIR): + return "superpowers" + return None + + +def map_foreign_paths(target: str | Path) -> LoopPaths | None: + """A ``LoopPaths`` view of a foreign run dir, or None if not foreign.""" + ws = Path(target).resolve() + if detect_foreign_layout(ws) != "superpowers": + return None + spec = _newest_md(ws / _SPECS_DIR) + plan = _newest_md(ws / _PLANS_DIR) + journal = next((ws / j for j in _JOURNALS if (ws / j).is_file()), None) + loop_dir = ws / ".loop" + return LoopPaths( + workspace=ws, + loop_dir=loop_dir, + manifest=loop_dir / "manifest.yaml", + state=loop_dir / "state.json", + tasks=ws / "TASKS.json", + runlog=journal if journal is not None else ws / "RUNLOG.md", + terminal=loop_dir / "terminal_state.json", + spec=spec if spec is not None else ws / "SPEC.md", + workflow=plan if plan is not None else ws / "WORKFLOW.md", + contract=ws / "loop-contract.md", + ) +``` + +- [ ] **Step 5.4: Wire the seam in `scripts/inspect_loop.py`** + +Three surgical edits, nothing else: + +(a) In the existing `try:` import block, extend: + +```python +try: + from loop.contract import TERMINAL_STATES, read_manifest + from loop.foreign import detect_foreign_layout, map_foreign_paths + from loop.paths import resolve_loop_paths +except ImportError: # pragma: no cover - direct script copy outside repo root +``` + +and inside the existing `except ImportError:` fallback add: + +```python + def detect_foreign_layout(_target): + return None + + def map_foreign_paths(_target): + return None +``` + +(b) Below the fallback block add the seam function: + +```python +def _resolve_paths(target): + """The path-resolution seam: a recognized foreign layout maps onto the + same LoopPaths surface; everything else resolves natively. The scoring + logic below is layout-blind — signals, weights, and credit tiers are + identical for native and foreign targets.""" + mapped = map_foreign_paths(target) + return mapped if mapped is not None else resolve_loop_paths(target) +``` + +(c) Replace the three call sites — in `_terminal_states_covered_from_contract` (`paths = resolve_loop_paths(loop)`), in `_evaluate_contract_checks` (`paths = resolve_loop_paths(loop)`), and in `inspect_loop`'s missing-states branch (`paths = resolve_loop_paths(loop)`) — with `paths = _resolve_paths(loop)`. + +(d) In `inspect_loop()`, right before `return`, add the labeling: + +```python + report = { + "target": str(loop), + "score": score, + "terminal_states_covered": covered, + "present": present, + "gaps": gaps, + "verdict": _verdict(score), + } + foreign = detect_foreign_layout(loop) + if foreign: + report["foreign_layout"] = foreign + report["advisory"] = True + return report +``` + +(refactor the existing literal-return into the `report` variable; no other line of the function changes). + +- [ ] **Step 5.5: Author the vendored fixture** (sanitized, fictional, no live network) + +`examples/superpowers-run/README.md`: + +```markdown +# superpowers-run — a vendored foreign-harness fixture + +A minimal, sanitized run directory in the layout the +[Superpowers](https://github.com/obra/superpowers) skills library leaves +behind (spec + plan under `docs/superpowers/`, a progress journal). All +content is fictional — it exists so `python3 -m loop inspect +examples/superpowers-run` can score a foreign layout read-only, and so +`docs/gap-reports/superpowers.md` has a checkable target. + +Superpowers is a **complement**, not a competitor: it is a skills library that +drives how an agent works; Loop Engineer is the contract layer that proves how +the work ended. The honest low score here is not a criticism — it measures +what a spec/plan/journal layout *structurally cannot prove* (no held-out gate, +no typed terminal record, no evidence trail), which is exactly what emitting +the contract adds. See the gap report for the item-by-item reading. +``` + +`examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md`: + +```markdown +# CSV dedupe — design + +> Fictional sample content for the vendored fixture. Not a real project. + +## Problem + +`import_contacts.py` writes duplicate rows when the same contact appears in +two source files with different casing. + +## Approach + +Normalize on a `(lower(email), lower(phone))` key before insert; keep the +first-seen row; log dropped duplicates to `dedupe.log`. + +## Success Criteria + +- Importing the two sample files yields 41 unique contacts (was 57 rows). +- Re-running the import is idempotent (0 new rows on the second run). +- Dropped duplicates are logged with their source line numbers. +``` + +`examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md`: + +```markdown +# CSV dedupe — implementation plan + +> Fictional sample content for the vendored fixture. Not a real project. + +## Task 1: normalization key + +- [x] Write `normalize_key(email, phone)` with lowercase + strip +- [x] Unit test with mixed-case fixtures + +## Task 2: idempotent import + +- [x] Skip insert when the key exists; count skips +- [x] Re-run the import; assert 0 new rows + +## Task 3: dedupe log + +- [x] Append dropped rows to `dedupe.log` with source line numbers +``` + +`examples/superpowers-run/.superpowers/sdd/progress.md`: + +```markdown +# progress + +- Implemented normalize_key and the idempotency guard; tests pass locally. +- Second import run inserted 0 rows. Marking the work complete. +``` + +(Deliberately: the journal *claims* completion with no gate run and no typed terminal — the exact self-report the inspector scores as weak. The plan text must NOT name the 7 terminal states, and no file mentions holdout/anticheat scripts.) + +- [ ] **Step 5.6: Run the tests until green** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_foreign_inspect.py scripts/test_inspect_loop.py scripts/test_example_runnable.py` +Expected: PASS, including the whole existing inspector suite (scorer untouched). If fixture score ≥50, remove signal-bearing phrasing from the fixture docs (it must stay honestly weak). + +- [ ] **Step 5.7: Full suite + commit** + +```bash +uv run --with pytest --with pyyaml --with jsonschema python -B -m pytest -q -p no:cacheprovider scripts +git add loop/foreign.py scripts/inspect_loop.py scripts/test_foreign_inspect.py examples/superpowers-run/ +git commit -m "feat(st4): foreign-harness inspect adapter — superpowers layout mapper + vendored fixture" +``` + +--- + +### Task 6: Checked-in gap report + +**Files:** +- Create: `docs/gap-reports/superpowers.md` + +**Interfaces:** +- Consumes: the fixture (Task 5), `reference/repo-os-contract.md` §14 checklist IDs A1–E1. + +- [ ] **Step 6.1: Write the report** + +`docs/gap-reports/superpowers.md` — required structure (write in full; ≤120 lines): + +1. Header + provenance box: evaluated against the vendored fixture `examples/superpowers-run/` only (fictional content, checked in); *no version-general claims about Superpowers itself*; date; the command to reproduce (`python3 -m loop inspect examples/superpowers-run`). +2. Complement framing paragraph (Superpowers = skills library driving how an agent works; the contract layer proves how work ended; "composes, doesn't compete"). +3. The §14 conformance table — every ID, verbatim from `reference/repo-os-contract.md` §14, evaluated against the fixture: + +| Item | What the standard requires | Fixture status | +|---|---|---| +| A1 | `.loop/manifest.yaml` valid against `manifest@1` incl. the canonical 7 `terminal_states` | **unmet** — no `.loop/` exists | +| A2 | `.loop/state.json` valid against `state@1` | **unmet** | +| A3 | `TASKS.json` valid against `tasks@1` (no dup ids; no evidence-free `done`) | **unmet** — plan checkboxes carry no evidence field at all | +| A4 | `RUNLOG.md` present | **unmet** — the journal (`progress.md`) narrates but is not an iteration log | +| B1 | exactly-one-of: no terminal vs valid terminal pair | **structurally unprovable** — the layout has no terminal record; "marking the work complete" lives in prose | +| B2 | `terminal@1` with `criteria_met`/`evidence`/`false_completion`; honest `Succeeded` rules | **structurally unprovable** | +| C1–C3 | receipts / repair / rollout validate when present | **absent** (nothing to check — and nothing to mine for FCR/RP) | +| D1–D2 | versioned `schema` keys; additive tolerance | **unmet** — no artifact carries a schema id | +| E1 | `doctor` lifecycle report consistent with B1 | **unmet** — `doctor` (correctly) refuses: no contract | + +4. `inspect` reading: the fixture scores weak (paste the actual score/gaps JSON from running it), labeled `foreign_layout: superpowers`, `advisory: true`. State plainly: the low score measures what the layout *cannot prove*, not the quality of the work or of Superpowers. +5. "What emitting the contract would add" — four bullets: a typed terminal instead of prose "complete"; a held-out gate making false completion *measurable*; an evidence trail (`criteria_met` → checks, verify bundles); FCR/RP derivable by `loop metrics`. Point at `loop.emit` (four calls) and `docs/integrations/langgraph.md` as the how. +6. Footer: this is the seed of the "inspect N public harnesses" scoreboard; contributions of further gap reports welcome (link the help-wanted issue from Task 7). + +- [ ] **Step 6.2: Verify the report's claims against the fixture** + +Run: `python3 -B -m loop inspect examples/superpowers-run` and paste the real JSON into §4 of the report. Every table row must be checkable against the fixture files. + +- [ ] **Step 6.3: Commit** + +```bash +git add docs/gap-reports/superpowers.md +git commit -m "docs(st4): superpowers gap report — §14 conformance read against the vendored fixture" +``` + +--- + +### Task 7: Second runnable example — `examples/flaky-test-triage/` + +**Files:** +- Create: `examples/flaky-test-triage/` — `README.md`, `SPEC.md`, `WORKFLOW.md`, `TASKS.json`, `RUNLOG.md`, `.loop/manifest.yaml`, `.loop/state.json`, `.loop/terminal_state.json`, `.loop/artifacts/verify-T1-iter1.json`, `.loop/artifacts/verify-T1.json`, `.loop/artifacts/holdout-verdict.json`, `.loop/repair/iter-002.json`, `target/jobs.py`, `target/test_visible.py`, `target/test_holdout.py`, `target/measure_stability.py`, `target/manifest.json`, `scripts/run-example`, `scripts/verify-fast`, `scripts/verify-full` +- Test: `scripts/test_flaky_example.py` + +**Interfaces:** +- Produces: a doctor-clean, gate-backed contract whose `loop metrics` scorecard has `repair_productivity == 1.0` (non-null — the differentiated pillar), `false_completion_rate == 0.0`, `evidence_backed: true`. +- Story: a genuinely flaky test (tie-order in a priority sort over an unordered set — deterministic per `PYTHONHASHSEED`, flaky across seeds) is triaged; the repair record carries `verification_before/after` scores anchored to a same-task red→green verify-bundle pair. + +- [ ] **Step 7.1: Write the failing test** + +Create `scripts/test_flaky_example.py`: + +```python +"""ST4 acceptance: the 2nd runnable example is doctor-clean and gate-backed, +and showcases the repair-record pillar — `loop metrics` derives a non-null RP +from a same-task red->green anchored repair. run-example re-derives the +committed gate verdict live from a foreign cwd.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +_EXAMPLE = _REPO / "examples" / "flaky-test-triage" + + +def _cli(cmd: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", cmd, str(_EXAMPLE)], + cwd=_REPO, capture_output=True, text=True, + ) + + +def test_doctor_clean(): + proc = _cli("doctor") + assert proc.returncode == 0, proc.stdout + proc.stderr + assert json.loads(proc.stdout)["ok"] is True + + +def test_metrics_derives_non_null_rp_and_clean_fcr(): + proc = _cli("metrics") + assert proc.returncode == 0, proc.stdout + proc.stderr + card = json.loads(proc.stdout) + assert card["repair_productivity"] == 1.0 + assert card["repair_passes"] == 1 + assert card["productive_repairs"] == 1 + assert card["false_completion_rate"] == 0.0 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["fcr_methods_agree"] is True + assert prov["rejected_records"] == [] + assert prov["unanchored_records"] == [] + assert prov["unmatched_verify"] == [] + + +def test_run_example_reproduces_the_gate_verdict_from_foreign_cwd(tmp_path): + proc = subprocess.run( + ["bash", str(_EXAMPLE / "scripts" / "run-example")], + cwd=tmp_path, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "BACKED by an independent" in proc.stdout + verdict = json.loads((_EXAMPLE / ".loop" / "artifacts" / "holdout-verdict.json").read_text()) + assert verdict["verdict"] == "Succeeded" + assert verdict["false_completion"] is False +``` + +- [ ] **Step 7.2: Run it to make sure it fails** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_flaky_example.py` +Expected: FAIL — example does not exist + +- [ ] **Step 7.3: Build the target (the repaired, deterministic state)** + +`target/jobs.py`: + +```python +"""Toy job scheduler: choose the next jobs to run, highest priority first.""" + + +def load_jobs(): + """Job records as an unordered set — insertion order is not meaningful.""" + return {("compact", 2), ("reindex", 2), ("backup", 1)} + + +def next_jobs(jobs): + """Job names ordered by priority (highest first), ties broken by name. + + The T1 repair: the sort key used to be priority alone, which left the + order of equal-priority jobs to set-iteration order — stable within one + process, different across PYTHONHASHSEED values. The (priority, name) + key makes the order a function of the data, not the interpreter state. + """ + return [name for name, _prio in sorted(jobs, key=lambda j: (-j[1], j[0]))] +``` + +`target/test_visible.py`: + +```python +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from jobs import load_jobs, next_jobs + + +def test_equal_priority_jobs_run_in_stable_order(): + assert next_jobs(load_jobs()) == ["compact", "reindex", "backup"] +``` + +`target/test_holdout.py`: + +```python +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from jobs import next_jobs + + +def test_order_is_independent_of_input_order(): + base = [("compact", 2), ("reindex", 2), ("backup", 1), ("prune", 3)] + rng = random.Random(7) + for _ in range(20): + shuffled = list(base) + rng.shuffle(shuffled) + assert next_jobs(shuffled) == ["prune", "compact", "reindex", "backup"] +``` + +`target/measure_stability.py`: + +```python +"""Deterministic flakiness probe: run the visible suite under 5 fixed +PYTHONHASHSEED values; the score is the passing fraction. Before the T1 +repair the tie order tracked set-iteration order, so some seeds failed +(committed red bundle: score 0.4); after it, 5/5 pass every time.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +SEEDS = ("0", "1", "2", "3", "4") + + +def main() -> int: + passed = 0 + for seed in SEEDS: + env = dict(os.environ, PYTHONHASHSEED=seed) + proc = subprocess.run( + [sys.executable, "-B", "-m", "pytest", "-q", "-p", "no:cacheprovider", + str(HERE / "test_visible.py")], + env=env, capture_output=True, text=True, cwd=HERE, + ) + passed += proc.returncode == 0 + score = passed / len(SEEDS) + print(json.dumps({"seeds": len(SEEDS), "passed": passed, "score": score})) + return 0 if passed == len(SEEDS) else 1 + + +if __name__ == "__main__": + sys.exit(main()) +``` + +`target/manifest.json`: + +```json +{ + "visible": [ + { "id": "visible-stability", "cmd": "python3 -B measure_stability.py" } + ], + "holdout": [ + { "id": "holdout-order-property", "cmd": "python3 -B -m pytest -q -p no:cacheprovider test_holdout.py" } + ] +} +``` + +Sanity-check the flakiness story is real before writing the frozen artifacts: temporarily change the sort key to `-j[1]` alone and run `python3 -B target/measure_stability.py` — expect a score < 1.0 (some seed orders `("reindex", ...)` before `("compact", ...)`). If all 5 seeds happen to pass with the buggy key, change the job names until at least one seed fails (keep names/expectations in the two test files and the frozen artifacts in sync), then restore the repaired key. Record the observed buggy score and use IT (not 0.4) as the `verification_before.score` everywhere below. + +- [ ] **Step 7.4: Write the contract + frozen artifacts** + +`.loop/manifest.yaml` — copy `examples/coverage-repair/.loop/manifest.yaml` verbatim, then change only: `loop: flaky-test-triage`, `inputs.goal: "Make the scheduler's job-order test deterministic across interpreter runs."`, the two `success_criteria` lines (`"1: visible suite passes under all 5 probe PYTHONHASHSEEDs — score 1.0 (target/measure_stability.py)"`, `"2: job order is a function of the data, independent of input order (target/test_holdout.py)"`), the constraints list (keep the "Do not edit tests… to manufacture a passing gate" line verbatim), and `outputs.repair_actions: .loop/repair/iter-002.json`. Keep the canonical 7 `terminal_states` block verbatim. + +`.loop/state.json`: + +```json +{ + "schema": "loop-engineer/state@1", + "project": "flaky-test-triage", + "iteration_id": 2, + "state": "terminal", + "plan_version": 1, + "active_task": "T1", + "best_score": 1.0, + "failure_mode": null, + "pending_approval": null, + "budget_remaining": { "time": "14m", "cost": "0.71usd" }, + "checkpoint_path": "", + "terminal_state": "Succeeded" +} +``` + +`TASKS.json`: + +```json +{ + "schema": "loop-engineer/tasks@1", + "project": "flaky-test-triage", + "tasks": [ + { + "id": "T1", + "title": "Make next_jobs tie-order deterministic (stable sort key, not set-iteration order)", + "status": "done", + "criterion_ref": "1", + "verify": "scripts/verify-full", + "depends_on": [], + "attempts": 2, + "evidence": ".loop/artifacts/verify-T1.json" + } + ], + "metadata": { + "created_at": "2026-07-08T10:02:00Z", + "updated_at": "2026-07-08T10:31:00Z", + "total_tasks": 1, + "completed": 1, + "failed": 0 + } +} +``` + +`.loop/artifacts/verify-T1-iter1.json` (the red bundle — use the OBSERVED buggy score from Step 7.3): + +```json +{ + "task": "T1", + "verify": "target/measure_stability.py", + "outcome": "FAIL", + "score": 0.4, + "iteration_id": 1, + "detail": "visible test passed under 2/5 probe seeds — tie order tracks set-iteration order" +} +``` + +`.loop/artifacts/verify-T1.json` (the green bundle): + +```json +{ + "task": "T1", + "verify": "target/measure_stability.py", + "outcome": "PASS", + "score": 1.0, + "iteration_id": 2, + "criteria": { "1": true }, + "detail": "visible test passed under 5/5 probe seeds after the (priority, name) sort key" +} +``` + +`.loop/repair/iter-002.json` (the pillar artifact — 7 canonical fields; before/after scores must equal the two bundles' scores exactly): + +```json +{ + "schema": "loop-engineer/repair@1", + "iteration_id": "2", + "attempt": 1, + "failure_mode": "flaky-fail", + "hypothesis": "next_jobs sorts by priority alone, so equal-priority jobs keep set-iteration order — stable within one interpreter, different across PYTHONHASHSEED values, which is why the visible test passes on some seeds and fails on others.", + "repair_action": "Changed the sort key from priority alone to (priority, name) in target/jobs.py so tie order is a function of the data. No edit to the tests, fixtures, probe script, or SPEC.md.", + "verification_before": { "verify_full": "FAIL", "metric": "visible_stability_score", "failing": ["seed 1", "seed 3", "seed 4"], "score": 0.4 }, + "verification_after": { "verify_full": "PASS", "metric": "visible_stability_score", "failing": [], "score": 1.0 }, + "remaining_delta": "none — 5/5 probe seeds green and the held-out order property holds", + "productive": true +} +``` + +(align the `failing` seed list with the observed run; keep `score` values byte-equal to the bundles). + +`RUNLOG.md` — two iteration blocks in the flagship's style; the load-bearing tokens are the headers and outcome tokens: + +```markdown +# RUNLOG.md — flaky-test-triage + +> Human-readable iteration history. Machine state lives in `.loop/state.json`. + +--- + +## Iteration 1 — 2026-07-08T10:08:00Z + +- **active_task:** `T1` — Make next_jobs tie-order deterministic +- **action:** Reproduced the flake: `target/measure_stability.py` runs the visible + test under 5 fixed PYTHONHASHSEED values; the buggy priority-only sort key + passed 2/5. +- **verify:** `scripts/verify-full` → FAIL — stability score 0.4 < 1.0 +- **outcome:** repair_triggered +- **evidence:** `.loop/artifacts/verify-T1-iter1.json` + +## Iteration 2 — 2026-07-08T10:24:00Z + +- **active_task:** `T1` +- **action:** Applied the repair from `.loop/repair/iter-002.json`: sort key + (priority, name) so tie order is a function of the data. +- **verify:** `scripts/verify-full` → PASS — stability score 1.0; held-out order + property green (holdout_gate verdict: Succeeded) +- **outcome:** task_passed +- **evidence:** `.loop/artifacts/verify-T1.json`, `.loop/artifacts/holdout-verdict.json` +``` + +`.loop/terminal_state.json`: + +```json +{ + "schema": "loop-engineer/terminal@1", + "project": "flaky-test-triage", + "state": "Succeeded", + "iteration_id": 2, + "terminated_at": "2026-07-08T10:31:00Z", + "goal": "Make the scheduler's job-order test deterministic across interpreter runs.", + "criteria_met": { "1": true, "2": true }, + "evidence": [".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + "false_completion": false, + "reason": "Stability score 1.0 under all 5 probe seeds (scripts/verify-full); held-out order-independence property green via a real holdout_gate run. The repair record carries the anchored 0.4 -> 1.0 delta.", + "total_iterations": 2, + "total_repair_attempts": 1 +} +``` + +`SPEC.md` and `WORKFLOW.md` — mirror the flagship's structure: SPEC with `## Success Criteria` (the two criteria above) and `## Constraints`; WORKFLOW naming the state machine, the approval gates line, `plan-then-execute` posture, and the canonical 7 terminal states verbatim (this is what gives `inspect` its terminal coverage). Adapt the flagship's text to the flaky-triage story; keep it under 60 lines each. + +- [ ] **Step 7.5: Write the runnable scripts** + +`scripts/verify-fast`: + +```bash +#!/usr/bin/env bash +# verify-fast — criterion 1 quick probe: the visible suite under one fixed seed. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +( cd "$EX/target" && PYTHONHASHSEED=3 python3 -B -m pytest -q -p no:cacheprovider test_visible.py ) +echo "verify-fast: PASS (visible test green under seed 3)" +``` + +`scripts/verify-full`: + +```bash +#!/usr/bin/env bash +# verify-full — criterion 1 (stability score over 5 seeds) + false-completion +# defense: the REAL repo held-out gate over the toy target's visible + holdout. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +REPO="$(cd "$EX/../.." && pwd)" + +bash "$EX/scripts/verify-fast" + +echo "== criterion 1: visible stability score == 1.0 over 5 probe seeds ==" +( cd "$EX/target" && python3 -B measure_stability.py ) + +echo "== false-completion defense: held-out gate over visible + holdout ==" +python3 "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target" +echo "verify-full: PASS (criteria 1+2 verified; holdout gate green)" +``` + +`scripts/run-example`: + +```bash +#!/usr/bin/env bash +# run-example — re-derive the committed holdout verdict from a LIVE gate run, +# then check the committed terminal claim against it. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +REPO="$(cd "$EX/../.." && pwd)" + +python3 -B "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target" \ + > "$EX/.loop/artifacts/holdout-verdict.json" + +python3 - "$EX" <<'PY' +import json, sys +from pathlib import Path + +example = Path(sys.argv[1]) +verdict = json.loads((example / ".loop/artifacts/holdout-verdict.json").read_text()) +terminal = json.loads((example / ".loop/terminal_state.json").read_text()) +assert verdict["verdict"] == "Succeeded", verdict +assert verdict["false_completion"] is False, verdict +assert terminal["false_completion"] is False +print("terminal claim BACKED by an independent holdout_gate run (verdict: Succeeded)") +PY +``` + +`chmod +x` all three. Generate the committed `.loop/artifacts/holdout-verdict.json` by running `bash examples/flaky-test-triage/scripts/run-example` once — never hand-type it (metrics validates it structurally as a real `decide()` output). + +`README.md`: what the example shows (a real flaky-test triage; the repair record + RP pillar; how `loop metrics` derives RP 1.0 and FCR 0.0 from the on-disk evidence), the three commands (`python3 -m loop doctor|inspect|metrics examples/flaky-test-triage`), and `bash scripts/run-example` to re-derive the verdict live. + +- [ ] **Step 7.6: Run tests until green** + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_flaky_example.py` +Expected: 3 PASS. Common failure modes: RP null → the repair record's scores don't byte-match the bundles, or bundle `iteration_id`s don't order red-before-green; `evidence_backed` false → the verdict artifact isn't the verbatim `decide()` output. + +Also run: `python3 -B -m loop inspect examples/flaky-test-triage` — expect `verdict: strong` (verify surface invokes the gate; 7/7 terminal states in WORKFLOW). Not a hard gate, but investigate if weak. + +- [ ] **Step 7.7: Full suite + commit** + +```bash +uv run --with pytest --with pyyaml --with jsonschema python -B -m pytest -q -p no:cacheprovider scripts +git add examples/flaky-test-triage/ scripts/test_flaky_example.py +git commit -m "feat(st4): 2nd runnable example — flaky-test-triage showcases repair records + non-null RP" +``` + +--- + +### Task 8: Contributor funnel — issue drafts + CONTRIBUTING section + +**Files:** +- Create: `docs/contributing/issues/01-good-first-qw9-trigger-phrases.md` +- Create: `docs/contributing/issues/02-good-first-qw10-self-eval-labels.md` +- Create: `docs/contributing/issues/03-good-first-emit-metrics-vocabulary.md` +- Create: `docs/contributing/issues/04-help-wanted-openhands-recipe.md` +- Create: `docs/contributing/issues/05-help-wanted-ruflo-recipe.md` +- Create: `docs/contributing/issues/06-help-wanted-gap-reports.md` +- Modify: `CONTRIBUTING.md` (add the "Start here" section after "Ground rule") + +- [ ] **Step 8.1: Write the six issue drafts** + +Each draft file starts with an HTML comment header the filing step parses by convention: + +```markdown + + +``` + +**01** — title `Trigger-phrase disambiguation batch (3 LOW fixes)`, labels `good first issue`. Body: the three fixes verbatim from the backlog QW9 — (a) `skills/loop-evals/SKILL.md` and `skills/loop-inspector/SKILL.md` both anchor bare "grade"; make the noun part of each phrase (evals: "grade a run's outcome against its SPEC"; inspector: "grade this harness/contract's readiness"); (b) trim `loop-evals` frontmatter description into the ~400–510-char sibling band by moving capability prose to the body; (c) `skills/loop-run/SKILL.md` opens with bare "run the loop" — qualify it ("run the agent loop"). **Gate that proves the fix:** `python3 scripts/validate_frontmatter.py` green + `python3 scripts/self_eval.py` green. + +**02** — title `Label self_eval terminal/repair/eval checks honestly as doc-completeness`, labels `good first issue`. Body: QW10 verbatim intent — `scripts/self_eval.py` `check_terminal_states`/`check_repair_fields`/`check_eval_layers_and_metrics` are substring-presence checks over `SKILL.md`; rename/comment them as documentation-completeness (not behavioral enforcement) and say so where self_eval is described as a gate (CONTRIBUTING + README's structural-check list). **Gate:** `python3 scripts/self_eval.py` green + the README accuracy assertions in `scripts/test_docs_claims.py`. + +**03** — title `Reconcile emit's iteration-outcome vocabulary with metrics' recognized tokens`, labels `good first issue`. Body: `loop/emit.py` `_ITERATION_OUTCOMES` accepts `approval_requested` and `replanned`, but `scripts/metrics.py` `_KNOWN_OUTCOME_TOKENS` recognizes neither — a RUNLOG written entirely through `emit` can still surface `provenance.unrecognized_outcomes`. Decide the canonical vocabulary and align (add the two tokens to metrics' honest-red set, or narrow emit); add a round-trip test (`emit.append_iteration` with every allowed outcome → `compute_metrics` reports `unrecognized_outcomes == []`). **Gate:** `python3 -m pytest scripts/test_metrics.py scripts/test_emit.py`. + +**04** — title `Integration recipe: OpenHands run → FCR gate`, labels `help wanted`. Body: design is written — ST3 spec §5.3 (`docs/superpowers/specs/2026-06-30-st3-integration-adapters.md`); follow the shipped LangGraph/Temporal recipes as the template (`loop/integrations.py` adapter, env-guarded e2e, CI job). **Gate:** doctor round-trip (`python3 -m loop doctor `) + a pinned false-completion invariant test (visible-green/holdout-red → `FailedUnverifiable`, `false_completion: true`, never `Succeeded`). + +**05** — title `Integration recipe: ruflo swarm → acceptance gate`, labels `help wanted`. Body: same shape as 04, design in ST3 spec §5.4; the swarm's terminal hook is the seam — no individual agent may declare the swarm done. **Gate:** same as 04. + +**06** — title `Foreign-harness gap reports — the inspect scoreboard pipeline`, labels `help wanted`. Body: `docs/gap-reports/superpowers.md` is the template; contribute reports for other public harness layouts (a vendored, sanitized fixture + the §14 A1–E1 table + complement framing; factual claims restricted to the fixture). **Gate:** `python3 -m loop inspect ` produces a scored report (exit criteria per `scripts/test_foreign_inspect.py` patterns) + the report follows the template's provenance rules. + +- [ ] **Step 8.2: Add the CONTRIBUTING funnel section** + +Insert after the "Ground rule" section: + +```markdown +## Start here — the contributor funnel + +Every open starter issue names **the gate that proves the fix** — a +deterministic command that is red before your change and green after. That is +the whole review bar (see the ground rule above). + +- [`good first issue`](https://github.com/SollanSystems/loop-engineer/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + — small, bounded, gate-verifiable fixes. +- [`help wanted`](https://github.com/SollanSystems/loop-engineer/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) + — integration recipes (OpenHands, ruflo — designs already written in + `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md`) and + foreign-harness gap reports (`docs/gap-reports/`). + +The contribution target for anything that emits or consumes contract +artifacts is the standard: `reference/repo-os-contract.md` — a harness that +satisfies the §14 conformance checklist (A1–E1) may claim it emits a +Loop-Engineer-conformant contract v1. Drafts for the seeded issues live in +`docs/contributing/issues/` and are filed on GitHub at release time. +``` + +- [ ] **Step 8.3: Gates + commit** + +Run: `python3 -B scripts/self_eval.py && uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_docs_claims.py scripts/test_docs_adoption.py` +Expected: green (docs checks unaffected but verify). + +```bash +git add docs/contributing/issues/ CONTRIBUTING.md +git commit -m "docs(st4): contributor funnel — six gate-backed issue drafts + CONTRIBUTING start-here section" +``` + +--- + +### Task 9: Release cut 0.8.0 (last commit of PR-B) + +**Files:** +- Modify: `CHANGELOG.md` +- Modify: `pyproject.toml` (line: `version = "0.7.0"` → `"0.8.0"`) +- Modify: `.claude-plugin/plugin.json` (`"version": "0.7.0"` → `"0.8.0"`) +- Modify: `README.md` (3 spots) +- Modify: `scripts/test_docs_version.py` + +- [ ] **Step 9.1: Update `scripts/test_docs_version.py` first (the failing test)** + +- Rename `test_plugin_version_is_0_7_0` → `test_plugin_version_is_0_8_0`; assert `"0.8.0"`. +- In `test_changelog_has_current_and_historical_entries`, add `assert "## 0.8.0" in changelog` at the top of the assert list (keep every historical assert, including `"## 0.7.0"`). + +Run: `uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider scripts/test_docs_version.py` +Expected: FAIL (versions still 0.7.0) + +- [ ] **Step 9.2: Apply the release edits** + +- `CHANGELOG.md`: rename `## Unreleased` → `## 0.8.0 — `, and extend the entry with the ST4 half: + +```markdown +**ST4 — contributor funnel.** `loop inspect` now recognizes a foreign +Superpowers-style run dir read-only (`loop/foreign.py` — a layout mapper onto +the existing `LoopPaths` seam; the M2/M3-hardened scorer is untouched and a +foreign harness with no gate and no terminal record scores honestly low). +The reading is checked in as `docs/gap-reports/superpowers.md` — the §14 +conformance checklist evaluated against a vendored, sanitized fixture +(`examples/superpowers-run/`). A second runnable example lands: +`examples/flaky-test-triage/` — doctor-clean, gate-backed, and the showcase +for repair records (`loop metrics` derives a non-null repair-productivity of +1.0 from its anchored red→green repair). Six gate-backed starter issues are +drafted under `docs/contributing/issues/` and filed at release; +CONTRIBUTING gains the start-here funnel. +``` + +- `pyproject.toml` + `.claude-plugin/plugin.json`: version → `0.8.0`. +- `README.md`: badge line `release-0.7.0-blue` → `release-0.8.0-blue`; action usage `SollanSystems/loop-engineer@v0.7.0` → `@v0.8.0`; Status section `- Version: \`0.7.0\`` → `0.8.0` and `- Release tag: \`v0.7.0\`` → `v0.8.0`. Also update Status's "Current reference example" line to name both runnable examples (`examples/coverage-repair`, `examples/flaky-test-triage`). + +- [ ] **Step 9.3: Full gate sweep** (same six commands as Step 4.2) + +Expected: all green. + +- [ ] **Step 9.4: Commit** + +```bash +git add CHANGELOG.md pyproject.toml .claude-plugin/plugin.json README.md scripts/test_docs_version.py +git commit -m "chore(release): 0.8.0 — composes the field (ST3 adapters + ST4 funnel)" +``` + +--- + +### Task 10: PR-B assembly — ship, then file the issues live + +- [ ] **Step 10.1: Fresh-checkout simulation** + +```bash +rm -rf /tmp/claude-1000/-mnt-c-Dev-projects-loop-engineer/*/scratchpad/le-fresh 2>/dev/null || true +mkdir -p "$SCRATCH/le-fresh" && git archive HEAD | tar -x -C "$SCRATCH/le-fresh" +uv run --with pytest --with pyyaml python -B -m pytest -q -p no:cacheprovider "$SCRATCH/le-fresh/scripts" +python3 -B -m py_compile "$SCRATCH"/le-fresh/loop/*.py "$SCRATCH"/le-fresh/scripts/*.py +``` +(where `$SCRATCH` is the session scratchpad dir; run pytest with `--rootdir` pointing at the fresh copy if collection misbehaves). Expected: green — proves no test depends on untracked state. + +- [ ] **Step 10.2: Push, open PR-B, CI green, merge** + +```bash +git push -u origin feat/v0.8.0-st4-funnel +gh pr create --title "feat(st4): contributor funnel + foreign-harness inspect adapter + 0.8.0 release cut" --body-file /tmp/claude-pr-b-body.md +gh pr checks --watch +gh pr merge --squash +``` + +PR body: spec link; the four deliverables (foreign adapter + fixture + gap report, flaky-test-triage example, funnel, release cut); acceptance-criteria checklist §5 items 4–7; test plan. + +- [ ] **Step 10.3: File the six issues live (authorized 2026-07-08)** + +After the merge, for each draft in `docs/contributing/issues/` (parse title/labels from the HTML comment header): + +```bash +gh issue create --title "" --label "<labels>" --body-file docs/contributing/issues/<file>.md +``` + +Verify: `gh issue list --label "good first issue"` shows 3, `gh issue list --label "help wanted"` shows 3. If a label doesn't exist, `gh label create` it first (default repos ship both). + +- [ ] **Step 10.4: Post-merge verification + report** + +On main: `git pull`, run the full gate sweep once, and confirm the milestone exit criteria (spec §5): recipes ×2 doctor-clean, LangGraph metrics-clean, foreign inspect scored report, gap report checked in, 2nd example doctor-clean, ≥4 issues live, CONTRIBUTING funnel, CI green incl. both recipe jobs, version 0.8.0. Report the human gates that remain open (PyPI pending-publisher registration; tagging `v0.8.0` at the release commit). + +--- + +## Self-Review (performed at authoring time) + +- **Spec coverage:** §1 decisions → Tasks 3 (Temporal), 2 (upgrade in place), 5 (Superpowers target), 8+10.3 (issues drafted then filed at merge), PR split (Tasks 4/10). §3.1 → Task 1. §3.2 → Task 2. §3.3 → Task 3. §3.4 → tests in Tasks 1–3 + file-discipline in Global Constraints. §4.1 → Tasks 5–6. §4.2 → Task 7. §4.3 → Task 8. §4.4 → Task 9. §5 acceptance → Steps 4.2/10.4. §6 risks → Steps 2.1/3.1 (API drift), 3.7 (dev-server CI), 5.1/5.5 (adapter honesty regression + fixture), 6.1 (naming/tone). +- **Placeholders:** none — every code step carries the actual content; the two deliberately build-time-resolved values (temporalio exception-constructor signatures, the observed buggy stability score) are named as explicit implementer verifications with the procedure to resolve them. +- **Type consistency:** `to_terminal_state` signature identical in Tasks 1/2/3; `EngineOutcome` fields consistent; `LoopPaths` construction in Task 5 matches `loop/paths.py`'s 10 fields; issue-draft header convention matches Step 10.3's parser. diff --git a/docs/superpowers/specs/2026-07-08-v0.8.0-composes-the-field-design.md b/docs/superpowers/specs/2026-07-08-v0.8.0-composes-the-field-design.md new file mode 100644 index 0000000..b73d7d0 --- /dev/null +++ b/docs/superpowers/specs/2026-07-08-v0.8.0-composes-the-field-design.md @@ -0,0 +1,221 @@ +# v0.8.0 — "Composes the field" (ST3 + ST4) + +> **Spec type:** design. +> **Closes:** ST3 (integration adapters) + ST4 (contributor funnel) — the whole +> v0.8.0 milestone (`docs/ROADMAP-v1.0.md`, `docs/superpowers/plans/2026-06-30-loop-engineer-v1.0-roadmap.md` §v0.8.0). +> **Builds on:** `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md` +> (the ST3 recipe design — normative for the mapping/precedence; this spec +> records only the deltas forced by code that shipped after it was written). +> **Date:** 2026-07-08 · **Status:** approved (operator decisions §1). + +--- + +## 1. Approved operator decisions + +1. **Second engine:** Temporal (the ST3 spec's flagship pair with LangGraph). + OpenHands / ruflo recipes become `help wanted` issues (§5.4), not milestone scope. +2. **LangGraph recipe:** upgrade `examples/langgraph-emit/` in place to the ST3 + bar rather than counting the B1 emit-only version as done. +3. **Foreign-harness target:** Superpowers-layout run dir (largest CC-native + name; real artifact layout available to design against). +4. **Issue filing:** issue bodies are drafted in PR-B and filed live on GitHub + at merge (authorized 2026-07-08). +5. **Delivery shape:** two PRs — **PR-A = ST3**, **PR-B = ST4 + 0.8.0 release + cut** (option B; reviewable units, release commit lands last). + +--- + +## 2. Deltas vs the 2026-06-30 ST3 spec + +The ST3 spec predates three things that shipped since; where they conflict, the +shipped reality wins: + +| 2026-06-30 spec said | Now | This milestone does | +|---|---|---| +| Recipes write `terminal_state.json` themselves; helper ships `write_terminal_state`/`append_receipt` writers | `loop/emit.py` (PR #21) is the enforcing writer API — G1 evidence rule + `criteria_met` guard at write time, atomic terminate (PR #28) | The adapter **assembles** the terminal body; all disk writes go through `emit.open_contract / append_iteration / append_receipt / terminate`. No second write path. | +| Helper module `loop_engineer.integrations`; docs home `reference/integrations/<engine>.md` | Package is `loop`; B1 established `docs/integrations/langgraph.md` + `examples/<recipe>/` | Helper = **`loop/integrations.py`**; docs stay in **`docs/integrations/`** (consistency with the shipped LangGraph recipe beats the stale path). | +| "Recipes want a versioned on-disk contract to target; gaps feed ST2" | ST2 shipped: `reference/repo-os-contract.md` is the normative standard (§0 versioning, §14 conformance checklist A1–E1) | Recipes and the gap report cite the standard by section; conformance claims use the §14 checklist IDs. | + +Everything else in the 2026-06-30 spec — the three-tier framing, the §3.1 +projection table, the §4 precedence (**safety → human → blocked → budget → +spec-gap → gate verdict**), the false-completion invariant, the +composes-not-competes discipline — carries over unchanged and is not restated. + +--- + +## 3. PR-A — ST3: adapter + two recipes + +### 3.1 `loop/integrations.py` (new, additive — the `emit.py` precedent) + +- `EngineOutcome` frozen dataclass: `reached_end`, `external_error`, + `budget_exhausted`, `human_abort`, `artifacts` (per ST3 spec §4). +- `to_terminal_state(outcome, gate_verdict, anticheat, criteria_met) -> dict`: + pure function, returns a `terminal@1` body. Implements the fixed precedence; + `Succeeded` is reachable **only** via `gate_verdict` green + anticheat clean + + ≥1 true criterion. `false_completion` is copied from the gate result, never + synthesized. +- **Zero engine imports; zero `scripts/` imports.** Gate and anticheat results + arrive as plain dicts (the recipe calls `scripts/holdout_gate.py` / + `scripts/anticheat_scan.py` and passes their JSON through). Pure stdlib, so + installing the helper never pulls LangGraph/Temporal/etc. +- Missing/empty anticheat or gate input maps fail-closed to + `FailedUnverifiable` (same posture as `holdout_gate` on an empty holdout set). +- Each recipe doc inlines the ~15-line copy-paste equivalent (zero-install + mode) — the installable module is convenience, not a requirement. + +### 3.2 LangGraph upgrade in place + +`examples/langgraph-emit/` + `docs/integrations/langgraph.md`: + +- The certify node routes through `EngineOutcome` + `to_terminal_state`, wiring + `holdout_gate.decide()` + the anticheat sweep, then writes via `emit`. +- Emits the verify bundle so `loop metrics` scores the run clean — closes the + recorded follow-up (recipe output scored FCR 1.0: doctor-pass ≠ metrics-clean + because no verify artifact backed the claim). +- Existing env-guarded e2e + `recipe-langgraph` CI job stay green; LangGraph + API re-verified against current docs before the snippet is finalized. + +### 3.3 Temporal recipe (new) + +`examples/temporal-certify/` + `docs/integrations/temporal.md`: + +- A `certify_activity` is the workflow's only path to a returned result + (activities do I/O; the workflow stays deterministic). Mapping + specializations: workflow `CancelledError` → `AbortedByHuman`; retry-policy + exhaustion on an external dependency → `FailedBlocked`; workflow timeout → + `FailedBudget` (ST3 spec §5.2). +- Env-guarded e2e against `temporalio`'s local dev server + a `recipe-temporal` + CI job mirroring `recipe-langgraph`. Temporal Python SDK API verified live + (Context7 / primary docs) before the snippet is finalized; the recipe records + a "verified against vX" note. + +### 3.4 Tests (both recipes + adapter) + +- **False-completion invariant, pinned per recipe:** visible-green/holdout-red + → `FailedUnverifiable` with `false_completion: true`, never `Succeeded`. +- **Precedence unit tests:** each of the 7 states reachable; `FailedSafety` and + `AbortedByHuman` beat a green gate; `Succeeded` unreachable without green + gate + clean anticheat + ≥1 true criterion. +- **Doctor round-trip:** each recipe's emitted contract passes + `python3 -m loop doctor` (the roadmap's ST3 verify), and the LangGraph run + passes `loop metrics` with zero unmatched claims. +- No file under `scripts/ schemas/ templates/ evals/` is modified; `loop/` + gains only the additive `integrations.py`. + +--- + +## 4. PR-B — ST4: contributor funnel + release cut + +### 4.1 Foreign-harness inspect adapter (read-only) + +- A **layout mapper**, not a scorer change: recognizes a Superpowers-style run + dir (`docs/superpowers/specs/*`, `docs/superpowers/plans/*`, journal/progress + artifacts) and maps it onto the surface `inspect_loop` already consumes via + the existing `read_manifest` / `resolve_loop_paths` seam. +- **The M2/M3-hardened scorer is not re-litigated.** No change to + `scripts/inspect_loop.py` scoring logic, signal tables, or credit tiers + (run-recorded / wired / none). A foreign harness with no holdout gate and no + terminal record scores honestly low — that *is* the product claim. +- Output labeled foreign/advisory. Roadmap verify: + `python3 -m loop inspect <foreign-harness-dir>` produces a scored report. +- **Vendored fixture:** a minimal, sanitized Superpowers-style run dir checked + in as the test fixture (no live network, no real project content). +- **Checked-in gap report** (`docs/gap-reports/superpowers.md`): the ST2 §14 + conformance checklist (A1–E1) evaluated against the fixture — which items a + Superpowers run satisfies, which it structurally cannot prove, and what + emitting the contract would add. Framed as *complement* (POSITIONING §7 + discipline: Superpowers is a skills library, not a rival). This is the seed + of the "inspect N public harnesses" scoreboard post (post-0.8.0). + +### 4.2 Second runnable example + +`examples/flaky-test-triage/` — full `.loop/` contract, doctor-clean, gate +-backed. Chosen over doc-migration because it showcases **repair records + the +RP metric** (the differentiated pillar): a flaky test is triaged, the repair +record carries `verification_before/after`, and `loop metrics` derives a +non-null RP. Roadmap verify: the example passes `doctor`. + +### 4.3 Contributor funnel + +- **≥4 issues**, each referencing the gate that proves the fix (backlog ST4 + acceptance). Drafted in the PR under `docs/contributing/issues/`, filed live + at merge. Seed list (final wording at plan time): + 1. `good first issue` — QW9: trigger-phrase disambiguation batch (3 LOW + fixes); gate = `self_eval.py`. + 2. `good first issue` — QW10: label `self_eval` checks honestly as + doc-completeness; gate = `self_eval.py` + README accuracy test. + 3. `good first issue` — emit↔metrics outcome vocabulary reconciliation + (`approval_requested`/`replanned` → `provenance.unrecognized_outcomes`); + gate = pytest metrics suite. + 4. `help wanted` — OpenHands recipe (ST3 spec §5.3 is the design); gate = + doctor round-trip + false-completion invariant test. + 5. `help wanted` — ruflo recipe (ST3 spec §5.4); same gate. + 6. `help wanted` — additional foreign-harness gap reports (scoreboard + pipeline); gate = `inspect` exit criteria + report template. +- **CONTRIBUTING.md** gains a "start here" funnel section linking the labels, + the gate-per-issue convention, and the standard (§14 conformance) as the + contribution target. + +### 4.4 Release cut 0.8.0 (in-PR, the 0.7.0 pattern) + +CHANGELOG 0.8.0; version bump `pyproject.toml` + `plugin.json` + README badge/ +Status; `test_docs_version` → 0.8.0. Human gates unchanged from 0.7.0 and still +open (PyPI pending-publisher registration; tag — recommendation stands to tag +v0.7.0 first or go straight to v0.8.0 at the release commit). + +--- + +## 5. Acceptance criteria (milestone exit) + +1. **ST3 (roadmap verify):** ≥2 recipes (LangGraph, Temporal), each producing + an on-disk contract that `python3 -m loop doctor <recipe-out>` accepts. +2. ST3 spec §7 criteria hold: false-completion invariant test per recipe; + `Succeeded` only via green gate + clean anticheat + ≥1 true criterion; + helper imports zero engine packages; copy-paste mode documented; every + recipe framed as complement. +3. LangGraph recipe additionally passes `loop metrics` with zero unmatched + claims (closes the FCR-1.0 follow-up). +4. **ST4 (roadmap verify):** `python3 -m loop inspect <foreign-harness-dir>` + produces a scored report against the vendored Superpowers-style fixture; + the gap report is checked in; the 2nd example passes `doctor`. +5. ≥4 labeled issues live on GitHub at merge, each naming its proving gate; + CONTRIBUTING links the funnel. +6. Suite gates green: full pytest (both jsonschema-present and structural + lanes), fresh-checkout simulation, `self_eval.py`, `validate_frontmatter.py`, + `py_compile`; CI green incl. `recipe-langgraph` + `recipe-temporal`. +7. 0.8.0 release commit is the last commit of PR-B; `test_docs_version` pins it. + +--- + +## 6. Risks + +- **Engine API drift** (ST3 spec §8): both snippets verified against live docs + at build time; recipes carry "verified against vX" notes. CI jobs pin the + tested versions. +- **Temporal dev server in CI:** `temporalio` downloads a dev-server binary at + first `start_local`; the CI job needs network + a cache step. If flaky, the + e2e stays env-guarded (skip-when-absent) and CI runs it in the dedicated + job only — the same containment as `recipe-langgraph`. +- **Foreign-adapter honesty:** the adapter must never manufacture credit — + mapping a Superpowers layout onto inspect's input surface is allowed; + synthesizing gate/verify artifacts is not. The gap report's low score for + unprovable items is the point, not a bug. Guarded by reusing the untouched + scorer + a regression test that the fixture's score contains no + run-recorded credit without an on-disk gate. +- **Superpowers naming:** the gap report names a third-party project. Tone is + complement-not-competitor; factual claims restricted to what the vendored + fixture shows; no version-general claims about Superpowers itself. + +--- + +## 7. Traceability + +| This spec | Grounded in | +|---|---| +| Milestone scope + verifies | `docs/ROADMAP-v1.0.md` §v0.8.0 · roadmap plan §v0.8.0 | +| Recipe design (mapping, precedence, engines) | `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md` | +| Writer API discipline | `loop/emit.py` (PR #21, #28) | +| Standard + conformance checklist | `reference/repo-os-contract.md` §0/§11–§14 (PR #31) | +| ST4 acceptance | `review/IMPROVEMENT-BACKLOG.md` ST4 (rank 19) | +| FCR-1.0 recipe follow-up | recorded follow-up, PR #21 session | +| Inspector honesty constraints | PR #30 (M2/M3, 4 adversarial rounds) | diff --git a/examples/langgraph-emit/README.md b/examples/langgraph-emit/README.md index 064677c..24d329f 100644 --- a/examples/langgraph-emit/README.md +++ b/examples/langgraph-emit/README.md @@ -1,34 +1,52 @@ -# LangGraph recipe — proof-of-done through `loop.emit` +# LangGraph recipe — gate the graph, then emit proof-of-done A runnable [LangGraph](https://github.com/langchain-ai/langgraph) graph whose -**terminal node writes the loop contract** — evidence-backed state the `loop` -CLI can independently validate. LangGraph keeps its own runtime; `loop.emit` is -a pure-stdlib writer that refuses to record a dishonest result. +`END` is reachable **only through a `certify` node**. LangGraph keeps its own +runtime; Loop Engineer adds the contract/proof tier above it — evidence-backed +state the `loop` CLI can independently validate and score. ## What it shows -`graph_example.py` runs three plain-function nodes — `do_work` writes -`artifact.txt`, `verify` re-reads it from disk, and `conclude` records the -outcome: +`graph_example.py` runs two plain-function nodes: -- On a real pass, `conclude` calls `emit.terminate(..., state="Succeeded", - evidence=["artifact.txt"])`. -- A lying `Succeeded` — no evidence, or no met criterion — raises `EmitError` - **before anything hits disk**. That is the same cross-check `loop doctor` - enforces, applied at write time. +- `do_work` writes `artifact.txt`. +- `certify` — the only edge into `END` — runs the same **visible + withheld + holdout** split the loop optimized against through the real `holdout_gate.decide` + and `anticheat_scan.scan`, projects the verdict through `to_terminal_state`, + and records it via `loop.emit`. It writes two evidence artifacts a scorecard + can join: the verbatim gate verdict (`holdout-verdict.json`) and a verify + bundle (`verify-T1.json`). + +On a real pass the terminal is `Succeeded` with evidence, and `loop metrics` +scores the run clean: `false_completion_rate 0.0`, `evidence_backed: true`, the +two FCR methods agree. + +### The `--sabotage-holdout` false-completion demo + +```bash +python graph_example.py sabotaged-run/ --sabotage-holdout +``` + +`do_work` now writes output that passes the **visible** check (the file exists) +but fails the **holdout** (the content is wrong). That is the measurable +false-completion event: the terminal becomes `FailedUnverifiable` with +`false_completion: true` — **never** `Succeeded`. The dishonest completion is +recorded, not laundered. ## Run it ```bash pip install loop-engineer langgraph python graph_example.py demo-run/ -loop doctor demo-run/ # -> {"ok": true, ...} +loop doctor demo-run/ # -> {"ok": true, ...} +loop metrics demo-run/ # -> clean scorecard ``` -`demo-run/.loop/terminal_state.json` ends `Succeeded` with `evidence`; `loop -doctor` validates it independently of the graph that wrote it. +The gate tools (`holdout_gate`, `anticheat_scan`) resolve from `loop._resources` +— the wheel bundles them, so a plain `pip install` is enough; running from a +repo checkout picks them up from `scripts/` too. -## The 10-line integration +## The general pattern -The general pattern (any graph, any terminal node) lives in +The complement framing and the copy-paste (zero-install) projection live in [`docs/integrations/langgraph.md`](../../docs/integrations/langgraph.md). diff --git a/examples/langgraph-emit/graph_example.py b/examples/langgraph-emit/graph_example.py index b42f602..a1625c1 100644 --- a/examples/langgraph-emit/graph_example.py +++ b/examples/langgraph-emit/graph_example.py @@ -1,14 +1,20 @@ -"""A minimal LangGraph graph that ships proof-of-done through loop.emit. +"""A LangGraph graph whose END is reachable only through a certify node. -The graph does real (tiny) work, verifies it from the filesystem, and the -terminal node records the outcome via emit.terminate(...) — which refuses an -evidence-free Succeeded. Run: +The certify node runs the SAME split the loop optimized against — a visible +check plus a WITHHELD holdout check — through the real holdout gate and +anticheat scan, projects the graph's terminal through loop.integrations, and +records the result via loop.emit (which refuses a dishonest Succeeded). - python graph_example.py <fresh-workspace-dir> + python graph_example.py <fresh-workspace-dir> [--sabotage-holdout] + +--sabotage-holdout makes do_work write output that passes the visible check +but fails the holdout — the measurable false-completion event: the terminal +becomes FailedUnverifiable with false_completion: true, never Succeeded. """ from __future__ import annotations +import json import sys from pathlib import Path from typing import TypedDict @@ -16,67 +22,108 @@ from langgraph.graph import END, START, StateGraph from loop import emit +from loop._resources import tools_dir +from loop.integrations import EngineOutcome, to_terminal_state + +sys.path.insert(0, str(tools_dir())) +import anticheat_scan # noqa: E402 +import holdout_gate # noqa: E402 + +EXPECTED = "hello from langgraph\n" class State(TypedDict): workspace: str - verified: bool + sabotage: bool + terminal_state: str def do_work(state: State) -> dict: out = Path(state["workspace"]) / "artifact.txt" - out.write_text("hello from langgraph\n", encoding="utf-8") + out.write_text("HELLO stub\n" if state["sabotage"] else EXPECTED, encoding="utf-8") return {} -def verify(state: State) -> dict: - artifact = Path(state["workspace"]) / "artifact.txt" - ok = artifact.is_file() and "hello" in artifact.read_text(encoding="utf-8") - return {"verified": ok} - - -def conclude(state: State) -> dict: - ws = state["workspace"] - passed = state["verified"] +def certify(state: State) -> dict: + ws = Path(state["workspace"]) + artifact = ws / "artifact.txt" + + # 1. The gate: visible = what the loop optimized against; holdout = withheld. + visible = [{"id": "artifact-exists", "passed": artifact.is_file()}] + holdout = [{ + "id": "artifact-content", + "passed": artifact.is_file() and artifact.read_text(encoding="utf-8") == EXPECTED, + }] + gate = holdout_gate.decide(visible, holdout) + ac = anticheat_scan.scan(diff_text="", trajectory=[str(artifact)]) + + # 2. Evidence artifacts: the gate verdict + a verify bundle metrics can join. + art_dir = ws / ".loop" / "artifacts" + art_dir.mkdir(parents=True, exist_ok=True) + (art_dir / "holdout-verdict.json").write_text(json.dumps(gate, indent=2) + "\n", encoding="utf-8") + bundle = { + "task": "T1", + "verify": "certify node — holdout_gate.decide over visible+holdout", + "outcome": "PASS" if gate["verdict"] == "Succeeded" else "FAIL", + "iteration_id": 1, + "criteria": {"1": gate["verdict"] == "Succeeded"}, + } + (art_dir / "verify-T1.json").write_text(json.dumps(bundle, indent=2) + "\n", encoding="utf-8") + + # 3. Project the graph terminal into a typed state; write via emit only. + terminal = to_terminal_state( + outcome=EngineOutcome( + reached_end=True, + artifacts=[".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + ), + gate_verdict=gate, + anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + passed = terminal["state"] == "Succeeded" emit.append_iteration( ws, iteration_id=1, outcome="task_passed" if passed else "task_failed", - task_id="T1", actions=["wrote artifact.txt", "re-read and checked content"], - verify_cmd="verify node (filesystem re-read)", verify_outcome="pass" if passed else "fail", + task_id="T1", + actions=["wrote artifact.txt", "ran holdout_gate.decide + anticheat_scan.scan"], + verify_cmd="holdout_gate.decide(visible, holdout)", verify_outcome=gate["verdict"], + notes="verify bundle: verify-T1.json; gate verdict: holdout-verdict.json", ) - if passed: - emit.terminate( - ws, state="Succeeded", criteria_met={"1": True}, - evidence=["artifact.txt"], reason="artifact written and independently re-read", - iteration_id=1, - ) - else: - emit.terminate( - ws, state="FailedUnverifiable", criteria_met={"1": False}, - evidence=[], reason="verification failed", iteration_id=1, - ) - return {} + emit.append_receipt(ws, iteration_id=1, role="orchestrate", model="deterministic-demo", outcome="ok") + emit.terminate( + ws, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return {"terminal_state": terminal["state"]} -def main(workspace: str) -> int: +def main(workspace: str, sabotage: bool) -> int: emit.open_contract(workspace) + # The fresh scaffold's RUNLOG carries a placeholder example iteration (its + # unfilled outcome renders as "REPLACE"); start the demo's run history clean + # so `loop metrics` scores only the graph's own iterations. + (Path(workspace) / "RUNLOG.md").write_text( + f"# RUNLOG.md — {Path(workspace).name}\n", encoding="utf-8" + ) graph = ( StateGraph(State) .add_node(do_work) - .add_node(verify) - .add_node(conclude) + .add_node(certify) .add_edge(START, "do_work") - .add_edge("do_work", "verify") - .add_edge("verify", "conclude") - .add_edge("conclude", END) + .add_edge("do_work", "certify") + .add_edge("certify", END) # certify IS the only path to END .compile() ) - graph.invoke({"workspace": workspace, "verified": False}) - print(f"contract emitted at {workspace}/.loop — run: python3 -m loop doctor {workspace}") - return 0 + result = graph.invoke({"workspace": workspace, "sabotage": sabotage, "terminal_state": ""}) + print(f"terminal: {result['terminal_state']} — validate: python3 -m loop doctor {workspace}") + return 0 if result["terminal_state"] == "Succeeded" else 1 if __name__ == "__main__": - if len(sys.argv) != 2: - print("usage: python graph_example.py <fresh-workspace-dir>", file=sys.stderr) + args = sys.argv[1:] + sabotage = "--sabotage-holdout" in args + targets = [a for a in args if not a.startswith("--")] + if len(targets) != 1: + print("usage: python graph_example.py <fresh-workspace-dir> [--sabotage-holdout]", file=sys.stderr) raise SystemExit(2) - raise SystemExit(main(sys.argv[1])) + raise SystemExit(main(targets[0], sabotage)) diff --git a/examples/temporal-certify/README.md b/examples/temporal-certify/README.md new file mode 100644 index 0000000..7ce6bcf --- /dev/null +++ b/examples/temporal-certify/README.md @@ -0,0 +1,67 @@ +# Temporal recipe — durable execution below, proof-of-done above + +A runnable [Temporal](https://github.com/temporalio/sdk-python) workflow whose +only path to a returned result is a **certify activity**. Temporal keeps its own +durable runtime (the workflow survives crashes, retries activities, resumes +where it left off); Loop Engineer adds the contract/proof tier above it — +evidence-backed state the `loop` CLI can independently validate and score. + +## What it shows + +`workflow_example.py` runs two activities under one workflow: + +- `do_work_activity` writes `artifact.txt`. +- `certify_activity` — whose return value is the workflow's **only** result — + runs the same **visible + withheld holdout** split the loop optimized against + through the real `holdout_gate.decide` and `anticheat_scan.scan`, projects the + verdict through `to_terminal_state`, and records it via `loop.emit`. It writes + two evidence artifacts a scorecard can join: the verbatim gate verdict + (`holdout-verdict.json`) and a verify bundle (`verify-T1.json`). + +Activities do the I/O; the `@workflow.defn` class stays deterministic. On a real +pass the terminal is `Succeeded` with evidence, and `loop metrics` scores the run +clean: `false_completion_rate 0.0`, `evidence_backed: true`, the two FCR methods +agree. + +### The `--sabotage-holdout` false-completion demo + +```bash +python workflow_example.py sabotaged-run/ --sabotage-holdout +``` + +`do_work_activity` now writes output that passes the **visible** check (the file +exists) but fails the **holdout** (the content is wrong). That is the measurable +false-completion event: the terminal becomes `FailedUnverifiable` with +`false_completion: true` — **never** `Succeeded`. The dishonest completion is +recorded, not laundered. + +### Host-side failure mapping + +Failures that never reach the certify activity are mapped off the +`WorkflowFailureError.cause` (`map_workflow_failure`): a workflow +`CancelledError` → `AbortedByHuman`, activity `RetryPolicy` exhaustion → +`FailedBlocked`, workflow timeout → `FailedBudget`. So a crash, cancel, or +budget cap still lands an honest terminal instead of an unwritten contract. + +## Run it + +```bash +pip install loop-engineer temporalio +python workflow_example.py demo-run/ # first run downloads a local Temporal dev server +loop doctor demo-run/ # -> {"ok": true, ...} +loop metrics demo-run/ # -> clean scorecard +``` + +Standalone mode starts a local Temporal dev server via +`temporalio.testing.WorkflowEnvironment.start_local()` — the **first** run +downloads the dev-server binary (network required); later runs reuse it. + +The gate tools (`holdout_gate`, `anticheat_scan`) resolve from `loop._resources` +— the wheel bundles them, so a plain `pip install` is enough; running from a +repo checkout picks them up from `scripts/` too. + +## The general pattern + +The complement framing, the signal→terminal-state mapping table, and the +copy-paste (zero-install) projection live in +[`docs/integrations/temporal.md`](../../docs/integrations/temporal.md). diff --git a/examples/temporal-certify/workflow_example.py b/examples/temporal-certify/workflow_example.py new file mode 100644 index 0000000..153d786 --- /dev/null +++ b/examples/temporal-certify/workflow_example.py @@ -0,0 +1,193 @@ +"""A Temporal workflow whose only path to a returned result is a certify +ACTIVITY — activities do I/O, the workflow stays deterministic. + +Temporal owns durability (the run survives crashes); Loop Engineer owns the +on-disk success/evidence truth. The certify activity runs the visible/holdout +split through the real holdout gate + anticheat scan, projects through +loop.integrations, and records the result via loop.emit. Host-side failure +mapping: cancellation -> AbortedByHuman, retry-policy exhaustion on an +external dependency -> FailedBlocked, workflow timeout -> FailedBudget. + + python workflow_example.py <fresh-workspace-dir> [--sabotage-holdout] + +(standalone mode starts a local Temporal dev server via +temporalio.testing.WorkflowEnvironment.start_local — first run downloads it) +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path + +from temporalio import activity, workflow +from temporalio.client import Client, WorkflowFailureError +from temporalio.common import RetryPolicy +from temporalio.exceptions import ActivityError, CancelledError +from temporalio.exceptions import TimeoutError as TemporalTimeoutError + +with workflow.unsafe.imports_passed_through(): + from loop import emit + from loop._resources import tools_dir + from loop.integrations import EngineOutcome, to_terminal_state + +sys.path.insert(0, str(tools_dir())) +with workflow.unsafe.imports_passed_through(): + import anticheat_scan # noqa: E402 + import holdout_gate # noqa: E402 + +EXPECTED = "hello from temporal\n" +TASK_QUEUE = "loop-engineer-certify-demo" + + +@dataclass +class WorkArgs: + workspace: str + sabotage: bool = False + hold_seconds: int = 0 + + +@activity.defn +async def do_work_activity(args: WorkArgs) -> str: + out = Path(args.workspace) / "artifact.txt" + out.write_text("HELLO stub\n" if args.sabotage else EXPECTED, encoding="utf-8") + return str(out) + + +@activity.defn +async def certify_activity(args: WorkArgs) -> dict: + ws = Path(args.workspace) + artifact = ws / "artifact.txt" + visible = [{"id": "artifact-exists", "passed": artifact.is_file()}] + holdout = [{ + "id": "artifact-content", + "passed": artifact.is_file() and artifact.read_text(encoding="utf-8") == EXPECTED, + }] + gate = holdout_gate.decide(visible, holdout) + ac = anticheat_scan.scan(diff_text="", trajectory=[str(artifact)]) + + # Evidence artifacts: the gate verdict + a verify bundle metrics can join — + # `loop metrics` scores a success claim as a false completion unless a green + # verify bundle backs it (same pair the LangGraph recipe writes). + art_dir = ws / ".loop" / "artifacts" + art_dir.mkdir(parents=True, exist_ok=True) + (art_dir / "holdout-verdict.json").write_text(json.dumps(gate, indent=2) + "\n", encoding="utf-8") + bundle = { + "task": "T1", + "verify": "certify activity — holdout_gate.decide over visible+holdout", + "outcome": "PASS" if gate["verdict"] == "Succeeded" else "FAIL", + "iteration_id": 1, + "criteria": {"1": gate["verdict"] == "Succeeded"}, + } + (art_dir / "verify-T1.json").write_text(json.dumps(bundle, indent=2) + "\n", encoding="utf-8") + + terminal = to_terminal_state( + outcome=EngineOutcome( + reached_end=True, + artifacts=[".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + ), + gate_verdict=gate, anticheat=ac, + criteria_met={"1": gate["verdict"] == "Succeeded"}, + ) + passed = terminal["state"] == "Succeeded" + emit.append_iteration( + str(ws), iteration_id=1, outcome="task_passed" if passed else "task_failed", + task_id="T1", actions=["do_work_activity wrote artifact.txt", "certify_activity gated it"], + verify_cmd="holdout_gate.decide(visible, holdout)", verify_outcome=gate["verdict"], + notes="verify bundle: verify-T1.json; gate verdict: holdout-verdict.json", + ) + emit.append_receipt(str(ws), iteration_id=1, role="orchestrate", model="deterministic-demo", outcome="ok") + emit.terminate( + str(ws), state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return {"state": terminal["state"], "false_completion": terminal["false_completion"]} + + +@workflow.defn +class CertifiedGoalWorkflow: + @workflow.run + async def run(self, args: WorkArgs) -> dict: + if args.hold_seconds: + await asyncio.sleep(args.hold_seconds) # durable timer (cancel/timeout demos) + await workflow.execute_activity( + do_work_activity, args, + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy(maximum_attempts=2), + ) + return await workflow.execute_activity( + certify_activity, args, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=RetryPolicy(maximum_attempts=1), + ) + + +def map_workflow_failure(cause: BaseException | None) -> EngineOutcome: + """Pure projection of a WorkflowFailureError cause onto EngineOutcome.""" + if isinstance(cause, CancelledError): + return EngineOutcome(reached_end=False, human_abort=True) + if isinstance(cause, TemporalTimeoutError): + return EngineOutcome(reached_end=False, budget_exhausted=True) + if isinstance(cause, ActivityError): + return EngineOutcome(reached_end=False, external_error=f"activity retries exhausted: {cause}") + return EngineOutcome(reached_end=False, external_error=str(cause) or "unknown engine failure") + + +def certify_workflow_failure(workspace: str, cause: BaseException | None) -> dict: + """Terminate an already-opened contract from a workflow failure, honestly.""" + terminal = to_terminal_state( + outcome=map_workflow_failure(cause), gate_verdict={}, anticheat={}, + criteria_met={"1": False}, + ) + emit.terminate( + workspace, state=terminal["state"], criteria_met=terminal["criteria_met"], + evidence=terminal["evidence"], false_completion=terminal["false_completion"], + reason=terminal["reason"], iteration_id=1, + ) + return terminal + + +async def run_and_certify(client: Client, workspace: str, *, sabotage: bool, wf_id: str) -> dict: + emit.open_contract(workspace) + # The fresh scaffold's RUNLOG carries a placeholder example iteration (its + # unfilled outcome renders as "REPLACE"); start the demo's run history clean + # so `loop metrics` scores only the workflow's own iterations. + (Path(workspace) / "RUNLOG.md").write_text( + f"# RUNLOG.md — {Path(workspace).name}\n", encoding="utf-8" + ) + try: + return await client.execute_workflow( + CertifiedGoalWorkflow.run, WorkArgs(workspace=workspace, sabotage=sabotage), + id=wf_id, task_queue=TASK_QUEUE, + ) + except WorkflowFailureError as exc: + return certify_workflow_failure(workspace, exc.cause) + + +async def _amain(workspace: str, sabotage: bool) -> int: + from temporalio.testing import WorkflowEnvironment + from temporalio.worker import Worker + + async with await WorkflowEnvironment.start_local() as env: + async with Worker( + env.client, task_queue=TASK_QUEUE, + workflows=[CertifiedGoalWorkflow], + activities=[do_work_activity, certify_activity], + ): + result = await run_and_certify(env.client, workspace, sabotage=sabotage, wf_id="demo-run") + print(f"terminal: {result['state']} — validate: python3 -m loop doctor {workspace}") + return 0 if result["state"] == "Succeeded" else 1 + + +if __name__ == "__main__": + args = sys.argv[1:] + sabotage = "--sabotage-holdout" in args + targets = [a for a in args if not a.startswith("--")] + if len(targets) != 1: + print("usage: python workflow_example.py <fresh-workspace-dir> [--sabotage-holdout]", file=sys.stderr) + raise SystemExit(2) + raise SystemExit(asyncio.run(_amain(targets[0], sabotage))) diff --git a/loop/integrations.py b/loop/integrations.py new file mode 100644 index 0000000..7b3907f --- /dev/null +++ b/loop/integrations.py @@ -0,0 +1,125 @@ +"""Engine-outcome -> typed-terminal projection (ST3). + +A pure projection, never a runtime: recipes adapt their engine's native result +into an ``EngineOutcome``, pass the holdout-gate and anticheat results through +as plain dicts (``scripts/holdout_gate.py decide(...)`` / ``scripts/ +anticheat_scan.py scan(...)`` JSON), and this module assembles the +``terminal@1`` body. Every disk write stays in ``loop.emit``. + +The fixed precedence — safety -> human -> blocked -> budget -> spec-gap -> +gate verdict — means a gamed (FailedSafety) or human-killed (AbortedByHuman) +run can never launder itself into Succeeded. ``Succeeded`` is reachable ONLY +via a green gate verdict + anticheat clean of HIGH/CRITICAL findings + at +least one met criterion + non-empty evidence. ``false_completion`` is +copied out of the gate result, never synthesized. A missing or +structurally-empty gate/anticheat input fails closed to +``FailedUnverifiable`` — the same posture as ``holdout_gate`` on an empty +holdout set. + +Pure stdlib; imports no engine package and nothing from ``scripts/`` — so +installing this helper never pulls LangGraph/Temporal/etc. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Sequence + +TERMINAL_SCHEMA = "loop-engineer/terminal@1" + + +@dataclass(frozen=True) +class EngineOutcome: + """Engine-agnostic description of how a host run ended.""" + + reached_end: bool + external_error: str | None = None + budget_exhausted: bool = False + human_abort: bool = False + artifacts: Sequence[str] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "artifacts", tuple(str(a) for a in self.artifacts)) + + +def _valid_checks(checks: object) -> bool: + return ( + isinstance(checks, list) + and bool(checks) + and all(isinstance(c, dict) and "id" in c and isinstance(c.get("passed"), bool) for c in checks) + ) + + +def _valid_gate(gate: dict) -> bool: + """Structurally a ``holdout_gate.decide`` result: verdict + flag + the + per-check ``visible``/``holdout`` evidence arrays. A hand-typed stub with + no check evidence is not a gate run.""" + return ( + isinstance(gate.get("verdict"), str) + and isinstance(gate.get("false_completion"), bool) + and _valid_checks(gate.get("visible")) + and _valid_checks(gate.get("holdout")) + ) + + +def _valid_anticheat(anticheat: dict) -> bool: + return isinstance(anticheat.get("findings"), list) and "downgrade_to" in anticheat + + +def to_terminal_state( + outcome: EngineOutcome, + gate_verdict: dict | None, + anticheat: dict | None, + criteria_met: dict[str, bool | None], +) -> dict: + """Project an engine terminal + gate/anticheat evidence into a terminal@1 body. + + ``criteria_met`` maps each SPEC criterion id to the pass/fail of its mapped + check; ``None`` means the criterion has no mapped check at all -> + ``FailedSpecGap``. In the returned body ``None`` coerces to ``False`` + (unproven is not met). + """ + gate = gate_verdict if isinstance(gate_verdict, dict) else {} + ac = anticheat if isinstance(anticheat, dict) else {} + false_completion = gate.get("false_completion") is True + + def body(state: str, reason: str) -> dict: + return { + "schema": TERMINAL_SCHEMA, + "state": state, + "criteria_met": {str(k): v is True for k, v in criteria_met.items()}, + "evidence": list(outcome.artifacts), + "false_completion": false_completion, + "reason": reason, + } + + if _valid_anticheat(ac) and ac.get("downgrade_to") == "FailedSafety": + return body("FailedSafety", "anticheat: critical gate-tampering finding") + if outcome.human_abort: + return body("AbortedByHuman", "operator interrupt / human abort signal") + if outcome.external_error: + return body("FailedBlocked", f"unrecoverable external block: {outcome.external_error}") + if outcome.budget_exhausted: + return body("FailedBudget", "engine budget cap hit (steps/tokens/wall-clock/cost)") + unmapped = sorted(str(k) for k, v in criteria_met.items() if v is None) + if unmapped: + return body("FailedSpecGap", "criteria with no mapped check: " + ", ".join(unmapped)) + if not _valid_anticheat(ac): + return body("FailedUnverifiable", "no anticheat result — cannot certify (fail closed)") + if not _valid_gate(gate): + return body("FailedUnverifiable", "no holdout gate result — cannot certify (fail closed)") + if ac.get("downgrade_to") == "FailedUnverifiable": + return body("FailedUnverifiable", "anticheat: high-severity finding") + if gate["verdict"] != "Succeeded": + if false_completion: + return body("FailedUnverifiable", "visible passed but holdout failed — false completion") + return body("FailedUnverifiable", f"gate verdict {gate['verdict']!r} — cannot certify Succeeded") + if false_completion: + return body("FailedUnverifiable", "gate flags false_completion — refusing Succeeded") + if not any(v is True for v in criteria_met.values()): + return body("FailedUnverifiable", "green gate but no met criterion — cannot certify") + if not outcome.artifacts: + return body("FailedUnverifiable", "green gate but no evidence artifacts — cannot certify") + if not outcome.reached_end: + return body("FailedUnverifiable", "engine did not reach its own terminal signal") + return body("Succeeded", "holdout gate green, anticheat clean, criteria met with evidence") diff --git a/scripts/test_integrations.py b/scripts/test_integrations.py new file mode 100644 index 0000000..0b25a19 --- /dev/null +++ b/scripts/test_integrations.py @@ -0,0 +1,170 @@ +"""ST3 adapter contract: the projection from an engine outcome + gate/anticheat +evidence to one of the 7 typed terminal states. Pins the fixed precedence +(safety -> human -> blocked -> budget -> spec-gap -> gate verdict), the +false-completion invariant, and the structural unreachability of Succeeded +without a green gate + clean anticheat + a met criterion + evidence.""" + +from __future__ import annotations + +import importlib.util +import re +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from loop import emit # noqa: E402 +from loop.integrations import EngineOutcome, to_terminal_state # noqa: E402 + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, Path(__file__).parent / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _gate(visible_pass: bool, holdout_pass: bool) -> dict: + """A REAL holdout_gate.decide(...) verdict — never a hand-typed stub.""" + holdout_gate = _load("holdout_gate") + return holdout_gate.decide( + visible=[{"id": "v1", "passed": visible_pass}], + holdout=[{"id": "h1", "passed": holdout_pass}], + ) + + +_CLEAN_AC = {"findings": [], "clean": True, "downgrade_to": None} +_HIGH_AC = { + "findings": [{"signature": "hidden-answer-read", "severity": "high", "evidence": "x"}], + "clean": False, + "downgrade_to": "FailedUnverifiable", +} +_CRITICAL_AC = { + "findings": [{"signature": "gate-tampering", "severity": "critical", "evidence": "x"}], + "clean": False, + "downgrade_to": "FailedSafety", +} +_ENDED = dict(reached_end=True, artifacts=["a.txt"]) + + +def test_succeeded_via_green_gate_clean_anticheat_met_criterion(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True}) + assert body["state"] == "Succeeded" + assert body["false_completion"] is False + assert body["schema"] == "loop-engineer/terminal@1" + assert body["evidence"] == ["a.txt"] + + +def test_false_completion_invariant_visible_green_holdout_red(): + gate = _gate(True, False) + assert gate["false_completion"] is True # the real decide() flag + body = to_terminal_state(EngineOutcome(**_ENDED), gate, _CLEAN_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + assert body["state"] != "Succeeded" + assert body["false_completion"] is True # copied from the gate, not synthesized + + +def test_anticheat_critical_beats_green_gate(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CRITICAL_AC, {"1": True}) + assert body["state"] == "FailedSafety" + + +def test_human_abort_beats_green_gate(): + body = to_terminal_state( + EngineOutcome(reached_end=True, human_abort=True, artifacts=["a"]), + _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert body["state"] == "AbortedByHuman" + + +def test_external_error_maps_to_failed_blocked(): + body = to_terminal_state( + EngineOutcome(reached_end=False, external_error="credential missing"), + {}, {}, {"1": False}, + ) + assert body["state"] == "FailedBlocked" + assert "credential missing" in body["reason"] + + +def test_budget_exhausted_maps_to_failed_budget(): + body = to_terminal_state( + EngineOutcome(reached_end=False, budget_exhausted=True), {}, {}, {"1": False}, + ) + assert body["state"] == "FailedBudget" + + +def test_unmapped_criterion_maps_to_failed_spec_gap(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True, "2": None}) + assert body["state"] == "FailedSpecGap" + assert body["criteria_met"] == {"1": True, "2": False} # None coerces to False + + +def test_all_seven_states_reachable(): + reached = { + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, False), _CLEAN_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(reached_end=False, external_error="x"), {}, {}, {"1": False})["state"], + to_terminal_state(EngineOutcome(reached_end=False, budget_exhausted=True), {}, {}, {"1": False})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CRITICAL_AC, {"1": True})["state"], + to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": None})["state"], + to_terminal_state(EngineOutcome(reached_end=False, human_abort=True), {}, {}, {"1": False})["state"], + } + assert reached == { + "Succeeded", "FailedUnverifiable", "FailedBlocked", "FailedBudget", + "FailedSafety", "FailedSpecGap", "AbortedByHuman", + } + + +def test_missing_gate_or_anticheat_fails_closed(): + assert to_terminal_state(EngineOutcome(**_ENDED), None, _CLEAN_AC, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), {}, _CLEAN_AC, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), None, {"1": True})["state"] == "FailedUnverifiable" + assert to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), {}, {"1": True})["state"] == "FailedUnverifiable" + + +def test_succeeded_unreachable_without_met_criterion_or_evidence(): + no_criterion = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": False}) + assert no_criterion["state"] == "FailedUnverifiable" + no_evidence = to_terminal_state( + EngineOutcome(reached_end=True, artifacts=[]), _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert no_evidence["state"] == "FailedUnverifiable" + not_ended = to_terminal_state( + EngineOutcome(reached_end=False, artifacts=["a"]), _gate(True, True), _CLEAN_AC, {"1": True}, + ) + assert not_ended["state"] == "FailedUnverifiable" + + +def test_anticheat_high_downgrades_a_green_gate(): + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _HIGH_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + + +def test_not_ready_gate_cannot_certify(): + holdout_gate = _load("holdout_gate") + gate = holdout_gate.decide(visible=[], holdout=[]) # NotReady + body = to_terminal_state(EngineOutcome(**_ENDED), gate, _CLEAN_AC, {"1": True}) + assert body["state"] == "FailedUnverifiable" + + +def test_body_feeds_emit_terminate_round_trip(tmp_path): + ws = tmp_path / "run" + emit.open_contract(ws) + body = to_terminal_state(EngineOutcome(**_ENDED), _gate(True, True), _CLEAN_AC, {"1": True}) + path = emit.terminate( + ws, state=body["state"], criteria_met=body["criteria_met"], evidence=body["evidence"], + false_completion=body["false_completion"], reason=body["reason"], iteration_id=1, + ) + assert path.is_file() + + +def test_module_imports_no_engine_and_no_scripts(): + source = (_REPO / "loop" / "integrations.py").read_text(encoding="utf-8") + imports = [l for l in source.splitlines() if re.match(r"\s*(import|from)\s", l)] + for line in imports: + assert "langgraph" not in line and "temporalio" not in line and "scripts" not in line, line + # pure stdlib: the only allowed import roots + for line in imports: + assert re.match(r"\s*(from\s+(__future__|dataclasses|typing)\s+import|import\s+(dataclasses|typing))", line), line diff --git a/scripts/test_langgraph_recipe_st3.py b/scripts/test_langgraph_recipe_st3.py new file mode 100644 index 0000000..5428045 --- /dev/null +++ b/scripts/test_langgraph_recipe_st3.py @@ -0,0 +1,75 @@ +"""ST3 acceptance for the LangGraph recipe: the certify node routes through +EngineOutcome + to_terminal_state (gate + anticheat wired), the emitted +contract passes doctor, `loop metrics` scores the run clean (closes the +FCR-1.0 follow-up), and the false-completion invariant holds under sabotage. +Env-guarded: langgraph is a dev dependency of the example only.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("langgraph") + +REPO_ROOT = Path(__file__).resolve().parent.parent +EXAMPLE = REPO_ROOT / "examples" / "langgraph-emit" / "graph_example.py" + + +def _run_example(workspace: Path, *args: str) -> subprocess.CompletedProcess: + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) + return subprocess.run( + [sys.executable, "-B", str(EXAMPLE), str(workspace), *args], + cwd=workspace.parent, env=env, capture_output=True, text=True, + ) + + +def _cli(cmd: str, workspace: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", cmd, str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + + +def test_happy_path_is_doctor_clean_and_metrics_clean(tmp_path): + ws = tmp_path / "graph-run" + proc = _run_example(ws) + assert proc.returncode == 0, proc.stdout + proc.stderr + + doctored = _cli("doctor", ws) + assert doctored.returncode == 0, doctored.stdout + assert json.loads(doctored.stdout)["ok"] is True + + terminal = json.loads((ws / ".loop" / "terminal_state.json").read_text()) + assert terminal["state"] == "Succeeded" + assert terminal["false_completion"] is False + assert terminal["evidence"] + + metrics = _cli("metrics", ws) + assert metrics.returncode == 0, metrics.stdout + metrics.stderr + card = json.loads(metrics.stdout) + assert card["false_completion_rate"] == 0.0 + assert card["false_completions"] == 0 + assert card["iterations_claiming_success"] >= 1 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["unmatched_verify"] == [] + assert prov["unrecognized_outcomes"] == [] + assert prov["fcr_methods_agree"] is True + + +def test_sabotage_holdout_is_false_completion_never_succeeded(tmp_path): + ws = tmp_path / "graph-run-sabotaged" + proc = _run_example(ws, "--sabotage-holdout") + # the recipe exits non-zero on a non-Succeeded terminal, but still emits it + terminal = json.loads((ws / ".loop" / "terminal_state.json").read_text()) + assert terminal["state"] == "FailedUnverifiable" + assert terminal["state"] != "Succeeded" + assert terminal["false_completion"] is True, (terminal, proc.stdout, proc.stderr) + + doctored = _cli("doctor", ws) + assert json.loads(doctored.stdout)["ok"] is True # an honest failure is a valid contract diff --git a/scripts/test_temporal_recipe.py b/scripts/test_temporal_recipe.py new file mode 100644 index 0000000..e113b61 --- /dev/null +++ b/scripts/test_temporal_recipe.py @@ -0,0 +1,156 @@ +"""ST3 acceptance for the Temporal recipe: a certify ACTIVITY is the workflow's +only path to a returned result; the emitted contract passes doctor; the +false-completion invariant holds under sabotage; cancellation maps to +AbortedByHuman. Env-guarded: temporalio is a dev dependency of the example +only — the package stays zero-dependency. Uses asyncio.run (no pytest-asyncio). +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import subprocess +import sys +import uuid +from datetime import timedelta +from pathlib import Path + +import pytest + +pytest.importorskip("temporalio") + +from temporalio.client import WorkflowFailureError # noqa: E402 +from temporalio.testing import WorkflowEnvironment # noqa: E402 +from temporalio.worker import Worker # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +_EXAMPLE_DIR = REPO_ROOT / "examples" / "temporal-certify" +# The example dir must be importable by name: Temporal's workflow sandbox +# re-imports the workflow's module (CertifiedGoalWorkflow.__module__ == +# "workflow_example") through the normal import machinery to prepare a +# deterministic copy, so "workflow_example" has to be findable on sys.path. +if str(_EXAMPLE_DIR) not in sys.path: + sys.path.insert(0, str(_EXAMPLE_DIR)) + +_spec = importlib.util.spec_from_file_location("workflow_example", _EXAMPLE_DIR / "workflow_example.py") +recipe = importlib.util.module_from_spec(_spec) +# Register before exec (the stdlib "importing a source file directly" recipe): the +# example's `from __future__ import annotations` + @dataclass WorkArgs makes the +# dataclass machinery resolve KW_ONLY via sys.modules[cls.__module__]; an +# unregistered module makes that None -> AttributeError on py3.10-3.12. +sys.modules["workflow_example"] = recipe +_spec.loader.exec_module(recipe) + + +def _doctor(workspace: Path) -> dict: + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "doctor", str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + return json.loads(proc.stdout) + + +def _metrics(workspace: Path) -> dict: + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "metrics", str(workspace)], + cwd=REPO_ROOT, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + return json.loads(proc.stdout) + + +def _terminal(workspace: Path) -> dict: + return json.loads((workspace / ".loop" / "terminal_state.json").read_text()) + + +def test_recipe_end_to_end(tmp_path): + async def scenario(): + async with await WorkflowEnvironment.start_local() as env: + async with Worker( + env.client, + task_queue=recipe.TASK_QUEUE, + workflows=[recipe.CertifiedGoalWorkflow], + activities=[recipe.do_work_activity, recipe.certify_activity], + ): + await recipe.run_and_certify( + env.client, str(tmp_path / "happy"), sabotage=False, + wf_id=f"happy-{uuid.uuid4()}", + ) + await recipe.run_and_certify( + env.client, str(tmp_path / "sabotaged"), sabotage=True, + wf_id=f"sab-{uuid.uuid4()}", + ) + # Cancellation -> AbortedByHuman (mapped host-side) + ws_cancel = tmp_path / "cancelled" + recipe.emit.open_contract(ws_cancel) + handle = await env.client.start_workflow( + recipe.CertifiedGoalWorkflow.run, + recipe.WorkArgs(workspace=str(ws_cancel), sabotage=False, hold_seconds=30), + id=f"cancel-{uuid.uuid4()}", task_queue=recipe.TASK_QUEUE, + ) + await handle.cancel() + try: + await handle.result() + except WorkflowFailureError as exc: + recipe.certify_workflow_failure(str(ws_cancel), exc.cause) + else: # pragma: no cover - cancellation must surface + raise AssertionError("expected WorkflowFailureError after cancel") + + asyncio.run(scenario()) + + happy = _terminal(tmp_path / "happy") + assert happy["state"] == "Succeeded" + assert happy["false_completion"] is False + assert happy["evidence"] + assert _doctor(tmp_path / "happy")["ok"] is True + # The README advertises a clean scorecard — pin it (sibling ST3 invariant). + card = _metrics(tmp_path / "happy") + assert card["false_completion_rate"] == 0.0 + assert card["false_completions"] == 0 + assert card["iterations_claiming_success"] >= 1 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["unmatched_verify"] == [] + assert prov["unrecognized_outcomes"] == [] + assert prov["fcr_methods_agree"] is True + + sab = _terminal(tmp_path / "sabotaged") + assert sab["state"] == "FailedUnverifiable" + assert sab["state"] != "Succeeded" + assert sab["false_completion"] is True + assert _doctor(tmp_path / "sabotaged")["ok"] is True + + cancelled = _terminal(tmp_path / "cancelled") + assert cancelled["state"] == "AbortedByHuman" + assert _doctor(tmp_path / "cancelled")["ok"] is True + + +def test_map_workflow_failure_covers_blocked_and_budget(): + from temporalio.exceptions import ActivityError, TimeoutError as TemporalTimeoutError + + blocked = recipe.map_workflow_failure( + ActivityError("activity failed", *_activity_error_extra_args()) + if False else _make(ActivityError, "activity failed") + ) + assert blocked.external_error + budget = recipe.map_workflow_failure(_make(TemporalTimeoutError, "workflow timeout")) + assert budget.budget_exhausted is True + + +def _make(exc_type, message): + """Best-effort construction of a temporalio failure for the pure mapper. + If a class needs richer args in the installed version, fall back to a + minimal subclass instance carrying only the type identity.""" + try: + return exc_type(message) + except TypeError: + stub = type(exc_type.__name__, (exc_type,), {"__init__": lambda self: None}) + return stub() + + +def _activity_error_extra_args(): # placeholder for versions needing more args + return ()