From b1525d51317014de92c2128c52647a29a853c5fa Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 20:56:58 -0400 Subject: [PATCH 01/10] =?UTF-8?q?feat(st4):=20foreign-harness=20inspect=20?= =?UTF-8?q?adapter=20=E2=80=94=20superpowers=20layout=20mapper=20+=20vendo?= =?UTF-8?q?red=20fixture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.superpowers/sdd/progress.md | 4 + examples/superpowers-run/README.md | 15 ++++ .../plans/2026-07-08-csv-dedupe.md | 17 ++++ .../specs/2026-07-08-csv-dedupe-design.md | 19 +++++ loop/foreign.py | 69 +++++++++++++++ scripts/inspect_loop.py | 30 ++++++- scripts/test_foreign_inspect.py | 84 +++++++++++++++++++ 7 files changed, 234 insertions(+), 4 deletions(-) create mode 100644 examples/superpowers-run/.superpowers/sdd/progress.md create mode 100644 examples/superpowers-run/README.md create mode 100644 examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md create mode 100644 examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md create mode 100644 loop/foreign.py create mode 100644 scripts/test_foreign_inspect.py diff --git a/examples/superpowers-run/.superpowers/sdd/progress.md b/examples/superpowers-run/.superpowers/sdd/progress.md new file mode 100644 index 0000000..f05701a --- /dev/null +++ b/examples/superpowers-run/.superpowers/sdd/progress.md @@ -0,0 +1,4 @@ +# progress + +- Implemented normalize_key and the idempotency guard; tests pass locally. +- Second import run inserted 0 rows. Marking the work complete. diff --git a/examples/superpowers-run/README.md b/examples/superpowers-run/README.md new file mode 100644 index 0000000..4ff6477 --- /dev/null +++ b/examples/superpowers-run/README.md @@ -0,0 +1,15 @@ +# superpowers-run — a vendored foreign-harness fixture + +A minimal, sanitized run directory in the layout the +[Superpowers](https://github.com/obra/superpowers) skills library leaves +behind (spec + plan under `docs/superpowers/`, a progress journal). All +content is fictional — it exists so `python3 -m loop inspect +examples/superpowers-run` can score a foreign layout read-only, and so +`docs/gap-reports/superpowers.md` has a checkable target. + +Superpowers is a **complement**, not a competitor: it is a skills library that +drives how an agent works; Loop Engineer is the contract layer that proves how +the work ended. The honest low score here is not a criticism — it measures +what a spec/plan/journal layout *structurally cannot prove* (no held-out gate, +no typed terminal record, no evidence trail), which is exactly what emitting +the contract adds. See the gap report for the item-by-item reading. diff --git a/examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md b/examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md new file mode 100644 index 0000000..b667e48 --- /dev/null +++ b/examples/superpowers-run/docs/superpowers/plans/2026-07-08-csv-dedupe.md @@ -0,0 +1,17 @@ +# CSV dedupe — implementation plan + +> Fictional sample content for the vendored fixture. Not a real project. + +## Task 1: normalization key + +- [x] Write `normalize_key(email, phone)` with lowercase + strip +- [x] Unit test with mixed-case fixtures + +## Task 2: idempotent import + +- [x] Skip insert when the key exists; count skips +- [x] Re-run the import; assert 0 new rows + +## Task 3: dedupe log + +- [x] Append dropped rows to `dedupe.log` with source line numbers diff --git a/examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md b/examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md new file mode 100644 index 0000000..2b18d60 --- /dev/null +++ b/examples/superpowers-run/docs/superpowers/specs/2026-07-08-csv-dedupe-design.md @@ -0,0 +1,19 @@ +# CSV dedupe — design + +> Fictional sample content for the vendored fixture. Not a real project. + +## Problem + +`import_contacts.py` writes duplicate rows when the same contact appears in +two source files with different casing. + +## Approach + +Normalize on a `(lower(email), lower(phone))` key before insert; keep the +first-seen row; log dropped duplicates to `dedupe.log`. + +## Success Criteria + +- Importing the two sample files yields 41 unique contacts (was 57 rows). +- Re-running the import is idempotent (0 new rows on the second run). +- Dropped duplicates are logged with their source line numbers. diff --git a/loop/foreign.py b/loop/foreign.py new file mode 100644 index 0000000..07a43e8 --- /dev/null +++ b/loop/foreign.py @@ -0,0 +1,69 @@ +"""Read-only foreign-harness layout mapping (ST4). + +Recognizes a run directory laid out by a foreign harness (currently: the +Superpowers spec/plan convention) and maps it onto the same ``LoopPaths`` +surface the inspector already consumes. A MAPPER, never a scorer: it points +the existing signals at foreign files and never manufactures credit — a +harness with no holdout gate and no terminal record scores honestly low, +which is the point. Synthesizing gate or verify artifacts here would be +dishonest and is out of scope by design. + +Used by ``inspect`` only; ``doctor`` stays unmapped (a foreign dir has no +contract to validate, and saying otherwise would be a false completion). +""" + +from __future__ import annotations + +from pathlib import Path + +from .paths import LoopPaths + +_SPECS_DIR = "docs/superpowers/specs" +_PLANS_DIR = "docs/superpowers/plans" +_JOURNALS = (".superpowers/sdd/progress.md", "docs/superpowers/journal.md") + + +def _newest_md(directory: Path) -> Path | None: + """Newest markdown file by name — Superpowers files are date-prefixed, so + lexicographic order is chronological order.""" + if not directory.is_dir(): + return None + files = sorted(p for p in directory.glob("*.md") if p.is_file()) + return files[-1] if files else None + + +def detect_foreign_layout(target: str | Path) -> str | None: + """Name the foreign layout of ``target``, or None. + + A native contract (``.loop/state.json``) always wins — foreign mapping + never shadows a real repo-OS contract. + """ + ws = Path(target) + if (ws / ".loop" / "state.json").is_file(): + return None + if _newest_md(ws / _SPECS_DIR) or _newest_md(ws / _PLANS_DIR): + return "superpowers" + return None + + +def map_foreign_paths(target: str | Path) -> LoopPaths | None: + """A ``LoopPaths`` view of a foreign run dir, or None if not foreign.""" + ws = Path(target).resolve() + if detect_foreign_layout(ws) != "superpowers": + return None + spec = _newest_md(ws / _SPECS_DIR) + plan = _newest_md(ws / _PLANS_DIR) + journal = next((ws / j for j in _JOURNALS if (ws / j).is_file()), None) + loop_dir = ws / ".loop" + return LoopPaths( + workspace=ws, + loop_dir=loop_dir, + manifest=loop_dir / "manifest.yaml", + state=loop_dir / "state.json", + tasks=ws / "TASKS.json", + runlog=journal if journal is not None else ws / "RUNLOG.md", + terminal=loop_dir / "terminal_state.json", + spec=spec if spec is not None else ws / "SPEC.md", + workflow=plan if plan is not None else ws / "WORKFLOW.md", + contract=ws / "loop-contract.md", + ) diff --git a/scripts/inspect_loop.py b/scripts/inspect_loop.py index 7bbbc8f..5ad2e9c 100644 --- a/scripts/inspect_loop.py +++ b/scripts/inspect_loop.py @@ -68,6 +68,7 @@ try: from loop.contract import TERMINAL_STATES, read_manifest + from loop.foreign import detect_foreign_layout, map_foreign_paths from loop.paths import resolve_loop_paths except ImportError: # pragma: no cover - direct script copy outside repo root TERMINAL_STATES = ( @@ -98,6 +99,22 @@ class _Paths: return _Paths() + def detect_foreign_layout(_target): + return None + + def map_foreign_paths(_target): + return None + + +def _resolve_paths(target): + """The path-resolution seam: a recognized foreign layout maps onto the + same LoopPaths surface; everything else resolves natively. The scoring + logic below is layout-blind — signals, weights, and credit tiers are + identical for native and foreign targets.""" + mapped = map_foreign_paths(target) + return mapped if mapped is not None else resolve_loop_paths(target) + + # Bound the read of any single target file: the corpus is substring-matched # against fixed signals, so the head of a file is enough and an oversized file # can never exhaust memory. @@ -257,7 +274,7 @@ def _task_titles_all_placeholder(tasks: dict) -> bool: def _terminal_states_covered_from_contract(loop: Path) -> int: """Count terminal taxonomy coverage from contract-owned files only.""" - paths = resolve_loop_paths(loop) + paths = _resolve_paths(loop) manifest = read_manifest(paths.manifest) or {} states = manifest.get("terminal_states") if isinstance(manifest, dict) else None if isinstance(states, list): @@ -498,7 +515,7 @@ def _evaluate_contract_checks(loop: Path) -> dict[str, object]: README prose, so keyword stuffing cannot satisfy the loop contract. """ - paths = resolve_loop_paths(loop) + paths = _resolve_paths(loop) # SPEC/WORKFLOW resolve dual-location (.loop/ ∪ root) via resolve_loop_paths; # a committed single-file loop-contract.md is folded in as a contract-owned # source for the same signals. @@ -616,7 +633,7 @@ def inspect_loop(loop_dir: str) -> dict: if covered == len(TERMINAL_STATES): present.append(f"all {len(TERMINAL_STATES)} terminal states reachable") else: - paths = resolve_loop_paths(loop) + paths = _resolve_paths(loop) manifest = read_manifest(paths.manifest) or {} states = manifest.get("terminal_states") if isinstance(manifest, dict) else None if isinstance(states, list): @@ -646,7 +663,7 @@ def inspect_loop(loop_dir: str) -> dict: "wire a holdout/anti-cheat gate" ) - return { + report = { "target": str(loop), "score": score, "terminal_states_covered": covered, @@ -654,6 +671,11 @@ def inspect_loop(loop_dir: str) -> dict: "gaps": gaps, "verdict": _verdict(score), } + foreign = detect_foreign_layout(loop) + if foreign: + report["foreign_layout"] = foreign + report["advisory"] = True + return report def main(argv: list[str]) -> int: diff --git a/scripts/test_foreign_inspect.py b/scripts/test_foreign_inspect.py new file mode 100644 index 0000000..fcd5cba --- /dev/null +++ b/scripts/test_foreign_inspect.py @@ -0,0 +1,84 @@ +"""ST4: the foreign-harness inspect adapter is a LAYOUT MAPPER, not a scorer +change. It maps a Superpowers-style run dir onto the LoopPaths surface the +inspector already consumes; the M2/M3-hardened scorer is not re-litigated. A +foreign harness with no holdout gate and no terminal record scores honestly +low — the regression tests pin that no run-recorded credit appears without an +on-disk gate, and that doctor does NOT get the mapping (inspect-only).""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from loop.foreign import detect_foreign_layout, map_foreign_paths # noqa: E402 + +FIXTURE = _REPO / "examples" / "superpowers-run" +NATIVE = _REPO / "examples" / "coverage-repair" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, Path(__file__).parent / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_detects_superpowers_layout_and_not_native_contracts(): + assert detect_foreign_layout(FIXTURE) == "superpowers" + assert detect_foreign_layout(NATIVE) is None # a native contract always wins + assert detect_foreign_layout(_REPO / "examples" / "naive-loop") is None + + +def test_mapping_points_at_superpowers_artifacts(): + paths = map_foreign_paths(FIXTURE) + assert paths is not None + assert paths.workspace == FIXTURE.resolve() + assert paths.spec.name == "2026-07-08-csv-dedupe-design.md" + assert paths.workflow.name == "2026-07-08-csv-dedupe.md" + assert paths.runlog.name == "progress.md" + assert map_foreign_paths(NATIVE) is None + + +def test_inspect_scores_fixture_low_and_labels_it_foreign(): + inspect_loop = _load("inspect_loop") + report = inspect_loop.inspect_loop(str(FIXTURE)) + assert report["foreign_layout"] == "superpowers" + assert report["advisory"] is True + assert report["verdict"] == "weak" + assert report["score"] < 50 + # honesty regression: NO run-recorded credit without an on-disk gate + assert not any("(invoked)" in p for p in report["present"]), report["present"] + assert report["terminal_states_covered"] < 7 + + +def test_native_reports_carry_no_foreign_label(): + inspect_loop = _load("inspect_loop") + report = inspect_loop.inspect_loop(str(NATIVE)) + assert "foreign_layout" not in report + assert "advisory" not in report + assert report["verdict"] == "strong" # flagship unchanged — scorer untouched + + +def test_cli_inspect_produces_scored_foreign_report(): + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "inspect", str(FIXTURE)], + cwd=_REPO, capture_output=True, text=True, + ) + report = json.loads(proc.stdout) + assert report["foreign_layout"] == "superpowers" + assert isinstance(report["score"], int) + + +def test_doctor_does_not_get_the_mapping(): + proc = subprocess.run( + [sys.executable, "-B", "-m", "loop", "doctor", str(FIXTURE)], + cwd=_REPO, capture_output=True, text=True, + ) + assert proc.returncode != 0 # no .loop contract -> doctor honestly fails From e10616b00f9f913b53f9b77e3051e00dd2ead1c6 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 21:09:29 -0400 Subject: [PATCH 02/10] =?UTF-8?q?docs(st4):=20superpowers=20gap=20report?= =?UTF-8?q?=20=E2=80=94=20=C2=A714=20conformance=20read=20against=20the=20?= =?UTF-8?q?vendored=20fixture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/gap-reports/superpowers.md | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/gap-reports/superpowers.md diff --git a/docs/gap-reports/superpowers.md b/docs/gap-reports/superpowers.md new file mode 100644 index 0000000..5a7c235 --- /dev/null +++ b/docs/gap-reports/superpowers.md @@ -0,0 +1,107 @@ +# Gap report — Superpowers (foreign-harness inspect) + +> **Provenance.** Evaluated **only** against the vendored fixture +> [`examples/superpowers-run/`](../../examples/superpowers-run/) — fictional, +> sanitized content, checked in. No version-general claims about Superpowers +> itself are made or implied: every statement below is checkable against that +> one directory. +> **Date:** 2026-07-08. +> **Reproduce:** `python3 -m loop inspect examples/superpowers-run` + +## Composes, doesn't compete + +Superpowers is a skills library — it drives *how* an agent works. The fixture +vendors exactly the layout such a run leaves behind: a design spec under +`docs/superpowers/specs/`, a plan of `- [x]` checkboxes under +`docs/superpowers/plans/`, and a prose `progress.md` journal. Loop Engineer is +the contract layer — it proves *how the work ended*. The two **compose**: a +Superpowers-driven run can emit a Loop-Engineer contract at the finish line. +This report is not a criticism of Superpowers or of the (fictional) work in the +fixture; the low score below measures only what a spec/plan/journal layout +*structurally cannot prove*. + +## §14 conformance — read against the fixture + +The "What the standard requires" column is verbatim from +[`reference/repo-os-contract.md`](../../reference/repo-os-contract.md) §14. + +| Item | What the standard requires | Fixture status | +|---|---|---| +| A1 | `.loop/manifest.yaml` valid against `manifest@1` incl. the canonical 7 `terminal_states` | **unmet** — no `.loop/` exists | +| A2 | `.loop/state.json` valid against `state@1` | **unmet** | +| A3 | `TASKS.json` valid against `tasks@1` (no dup ids; no evidence-free `done`) | **unmet** — plan checkboxes carry no evidence field at all | +| A4 | `RUNLOG.md` present | **unmet** — the journal (`progress.md`) narrates but is not an iteration log | +| B1 | exactly-one-of: no terminal vs valid terminal pair | **structurally unprovable** — the layout has no terminal record; "marking the work complete" lives in prose | +| B2 | `terminal@1` with `criteria_met`/`evidence`/`false_completion`; honest `Succeeded` rules | **structurally unprovable** | +| C1–C3 | receipts / repair / rollout validate when present | **absent** (nothing to check — and nothing to mine for FCR/RP) | +| D1–D2 | versioned `schema` keys; additive tolerance | **unmet** — no artifact carries a schema id | +| E1 | `doctor` lifecycle report consistent with B1 | **unmet** — `doctor` (correctly) refuses: no contract | + +Each row is checkable against the fixture: `examples/superpowers-run/` contains +only `README.md`, `docs/superpowers/specs/2026-07-08-csv-dedupe-design.md`, +`docs/superpowers/plans/2026-07-08-csv-dedupe.md`, and +`.superpowers/sdd/progress.md`. There is no `.loop/`, no `schema:` key in any +file, and `progress.md`'s closing line is the prose "Marking the work +complete." — the exact false-completion surface B1/B2 exist to make typed. +Running `python3 -m loop doctor examples/superpowers-run` returns `ok: false` +with `lifecycle: unknown` and `missing_file` issues for every contract +artifact (E1). + +## `inspect` reading + +```json +{ + "target": "examples/superpowers-run", + "score": 12, + "terminal_states_covered": 0, + "present": [ + "defines verifiable success criteria" + ], + "gaps": [ + "no independent verification (verify-* script / TASKS verify command) — success is self-asserted", + "no approval gates declared for side-effects (destructive / secret / production / money)", + "no false-completion defense: no recorded holdout/anti-cheat invocation (a self-asserted false_completion flag or prose mention earns no credit)", + "no plan-then-execute discipline for untrusted/web reads (prompt-injection surface)", + "0/7 terminal states present — missing Succeeded, FailedUnverifiable, FailedBlocked, FailedBudget, FailedSafety, FailedSpecGap, AbortedByHuman (loop can end in a silent 'completed')" + ], + "verdict": "weak", + "foreign_layout": "superpowers", + "advisory": true +} +``` + +The fixture scores **weak** (`score: 12`, `terminal_states_covered: 0`), +labeled `foreign_layout: superpowers` and `advisory: true`. Read this plainly: +the low score measures what the layout *cannot prove*, not the quality of the +work, nor of Superpowers. `advisory: true` is the whole point — a foreign +layout is read for gaps, never graded as a failing contract. + +## What emitting the contract would add + +- **A typed terminal instead of prose "complete."** The `Succeeded`/`Failed*` + record replaces `progress.md`'s free-text "Marking the work complete." with + one of the canonical 7 states plus a `false_completion` boolean. +- **A held-out gate that makes false completion *measurable*.** An + anti-cheat/holdout invocation is what turns "tests pass locally" into a + third-party-checkable verdict — the missing `false-completion defense` gap. +- **An evidence trail.** `criteria_met` maps each success criterion to a check; + verify bundles under `.loop/artifacts/` back the terminal instead of a + self-assertion. +- **FCR / RP derivable by `loop metrics`.** With receipts and repair records on + disk, false-completion-rate and repair-productivity are *computed*, not + claimed (the C1–C3 "nothing to mine for FCR/RP" gap). + +The *how* is small: four `loop.emit` calls — `open_contract`, +`append_iteration`, `append_receipt`, `terminate` — at the run's finish line. +That path is worked end-to-end for a real engine in +[`docs/integrations/langgraph.md`](../integrations/langgraph.md). + +## This is a seed + +This is the first entry in an "inspect N public harnesses" scoreboard: read a +foreign layout read-only, name the gaps a contract would close, keep every +claim checkable against a vendored fixture. Contributions of further gap +reports are welcome — this file is the template. See the drafted contributor +issue +[`docs/contributing/issues/06-help-wanted-gap-reports.md`](../contributing/issues/06-help-wanted-gap-reports.md) +(`help wanted: gap reports`, filed on GitHub at release). From a531e8f84115a111706ad11ea08edd6f8998a7c0 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 21:17:10 -0400 Subject: [PATCH 03/10] =?UTF-8?q?docs(st4):=20gap=20report=20=E2=80=94=20s?= =?UTF-8?q?tandard=20column=20is=20condensed,=20not=20verbatim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/gap-reports/superpowers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gap-reports/superpowers.md b/docs/gap-reports/superpowers.md index 5a7c235..d8ad251 100644 --- a/docs/gap-reports/superpowers.md +++ b/docs/gap-reports/superpowers.md @@ -22,7 +22,7 @@ fixture; the low score below measures only what a spec/plan/journal layout ## §14 conformance — read against the fixture -The "What the standard requires" column is verbatim from +The "What the standard requires" column is condensed from [`reference/repo-os-contract.md`](../../reference/repo-os-contract.md) §14. | Item | What the standard requires | Fixture status | From 18aae3f5ec81a16078b9fdde4959bb7d198b6322 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 21:48:43 -0400 Subject: [PATCH 04/10] =?UTF-8?q?feat(st4):=202nd=20runnable=20example=20?= =?UTF-8?q?=E2=80=94=20flaky-test-triage=20showcases=20repair=20records=20?= =?UTF-8?q?+=20non-null=20RP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.loop/artifacts/holdout-verdict.json | 21 ++++ .../.loop/artifacts/verify-T1-iter1.json | 8 ++ .../.loop/artifacts/verify-T1.json | 9 ++ .../flaky-test-triage/.loop/manifest.yaml | 53 ++++++++++ .../.loop/repair/iter-002.json | 12 +++ examples/flaky-test-triage/.loop/state.json | 14 +++ .../.loop/terminal_state.json | 14 +++ examples/flaky-test-triage/README.md | 96 +++++++++++++++++++ examples/flaky-test-triage/RUNLOG.md | 25 +++++ examples/flaky-test-triage/SPEC.md | 55 +++++++++++ examples/flaky-test-triage/TASKS.json | 23 +++++ examples/flaky-test-triage/WORKFLOW.md | 77 +++++++++++++++ .../flaky-test-triage/scripts/run-example | 22 +++++ .../flaky-test-triage/scripts/verify-fast | 6 ++ .../flaky-test-triage/scripts/verify-full | 15 +++ examples/flaky-test-triage/target/jobs.py | 17 ++++ .../flaky-test-triage/target/manifest.json | 8 ++ .../target/measure_stability.py | 32 +++++++ .../flaky-test-triage/target/test_holdout.py | 15 +++ .../flaky-test-triage/target/test_visible.py | 9 ++ scripts/test_flaky_example.py | 55 +++++++++++ 21 files changed, 586 insertions(+) create mode 100644 examples/flaky-test-triage/.loop/artifacts/holdout-verdict.json create mode 100644 examples/flaky-test-triage/.loop/artifacts/verify-T1-iter1.json create mode 100644 examples/flaky-test-triage/.loop/artifacts/verify-T1.json create mode 100644 examples/flaky-test-triage/.loop/manifest.yaml create mode 100644 examples/flaky-test-triage/.loop/repair/iter-002.json create mode 100644 examples/flaky-test-triage/.loop/state.json create mode 100644 examples/flaky-test-triage/.loop/terminal_state.json create mode 100644 examples/flaky-test-triage/README.md create mode 100644 examples/flaky-test-triage/RUNLOG.md create mode 100644 examples/flaky-test-triage/SPEC.md create mode 100644 examples/flaky-test-triage/TASKS.json create mode 100644 examples/flaky-test-triage/WORKFLOW.md create mode 100644 examples/flaky-test-triage/scripts/run-example create mode 100644 examples/flaky-test-triage/scripts/verify-fast create mode 100644 examples/flaky-test-triage/scripts/verify-full create mode 100644 examples/flaky-test-triage/target/jobs.py create mode 100644 examples/flaky-test-triage/target/manifest.json create mode 100644 examples/flaky-test-triage/target/measure_stability.py create mode 100644 examples/flaky-test-triage/target/test_holdout.py create mode 100644 examples/flaky-test-triage/target/test_visible.py create mode 100644 scripts/test_flaky_example.py diff --git a/examples/flaky-test-triage/.loop/artifacts/holdout-verdict.json b/examples/flaky-test-triage/.loop/artifacts/holdout-verdict.json new file mode 100644 index 0000000..1fa9be7 --- /dev/null +++ b/examples/flaky-test-triage/.loop/artifacts/holdout-verdict.json @@ -0,0 +1,21 @@ +{ + "verdict": "Succeeded", + "reason": "visible and holdout gates both green", + "passed_visible": true, + "passed_holdout": true, + "false_completion": false, + "visible": [ + { + "id": "visible-stability", + "passed": true, + "returncode": 0 + } + ], + "holdout": [ + { + "id": "holdout-order-property", + "passed": true, + "returncode": 0 + } + ] +} diff --git a/examples/flaky-test-triage/.loop/artifacts/verify-T1-iter1.json b/examples/flaky-test-triage/.loop/artifacts/verify-T1-iter1.json new file mode 100644 index 0000000..7979f47 --- /dev/null +++ b/examples/flaky-test-triage/.loop/artifacts/verify-T1-iter1.json @@ -0,0 +1,8 @@ +{ + "task": "T1", + "verify": "target/measure_stability.py", + "outcome": "FAIL", + "score": 0.6, + "iteration_id": 1, + "detail": "visible test passed under 3/5 probe seeds — tie order tracks set-iteration order" +} diff --git a/examples/flaky-test-triage/.loop/artifacts/verify-T1.json b/examples/flaky-test-triage/.loop/artifacts/verify-T1.json new file mode 100644 index 0000000..0763e71 --- /dev/null +++ b/examples/flaky-test-triage/.loop/artifacts/verify-T1.json @@ -0,0 +1,9 @@ +{ + "task": "T1", + "verify": "target/measure_stability.py", + "outcome": "PASS", + "score": 1.0, + "iteration_id": 2, + "criteria": { "1": true }, + "detail": "visible test passed under 5/5 probe seeds after the (priority, name) sort key" +} diff --git a/examples/flaky-test-triage/.loop/manifest.yaml b/examples/flaky-test-triage/.loop/manifest.yaml new file mode 100644 index 0000000..6136e4e --- /dev/null +++ b/examples/flaky-test-triage/.loop/manifest.yaml @@ -0,0 +1,53 @@ +# .loop/manifest.yaml — the explicit operating contract for one loop +# Scaffolded by [[loop-contract]] from ADR.md. Canonical schema: reference/repo-os-contract.md §10. +loop: flaky-test-triage +schema: loop-engineer/manifest@1 + +inputs: + goal: "Make the scheduler's job-order test deterministic across interpreter runs." + success_criteria: + - "1: visible suite passes under all 5 probe PYTHONHASHSEEDs — score 1.0 (target/measure_stability.py)" + - "2: job order is a function of the data, independent of input order (target/test_holdout.py)" + constraints: + - "Do not modify the public next_jobs / load_jobs signatures." + - "No new third-party dependencies." + - "Do not edit tests, fixtures, golden files, or the verify scripts to manufacture a passing gate." + workspace_path: "./" + allowed_tools: [read, workspace-write] # NOT network, NOT external-side-effects + risk_profile: low # low | med | high + time_budget: "30m" + cost_budget: "1.00usd" + approval_policy: on_side_effects # never | on_side_effects | strict + +outputs: + plan: SPEC.md + task_queue: TASKS.json + current_state: .loop/state.json + verification_bundle: .loop/artifacts/ + repair_actions: .loop/repair/iter-002.json + terminal_state: .loop/terminal_state.json + lessons_learned: .loop/memory/lessons.md + +permissions: # least-privilege tiers + - read-only + - workspace-write + # network / external-side-effects / production-mutation are OFF for this loop + +approval_gates: # each pauses-and-resumes from run state + - destructive_commands + - secret_access + - production_changes + +policies: + repair_cap: 2 # then replan | revert | approve | terminate + plan_then_execute: false # set true for untrusted/web environments + verifier_gaming: hard_terminate_as_security_failure + +terminal_states: # the canonical 7, verbatim + - Succeeded + - FailedUnverifiable + - FailedBlocked + - FailedBudget + - FailedSafety + - FailedSpecGap + - AbortedByHuman diff --git a/examples/flaky-test-triage/.loop/repair/iter-002.json b/examples/flaky-test-triage/.loop/repair/iter-002.json new file mode 100644 index 0000000..cdbb0b1 --- /dev/null +++ b/examples/flaky-test-triage/.loop/repair/iter-002.json @@ -0,0 +1,12 @@ +{ + "schema": "loop-engineer/repair@1", + "iteration_id": "2", + "attempt": 1, + "failure_mode": "flaky", + "hypothesis": "next_jobs sorts by priority alone, so equal-priority jobs keep set-iteration order — stable within one interpreter, different across PYTHONHASHSEED values, which is why the visible test passes on some seeds and fails on others.", + "repair_action": "Changed the sort key from priority alone to (priority, name) in target/jobs.py so tie order is a function of the data. No edit to the tests, fixtures, probe script, or SPEC.md.", + "verification_before": { "verify_full": "FAIL", "metric": "visible_stability_score", "failing": ["seed 1", "seed 4"], "score": 0.6 }, + "verification_after": { "verify_full": "PASS", "metric": "visible_stability_score", "failing": [], "score": 1.0 }, + "remaining_delta": "none — 5/5 probe seeds green and the held-out order property holds", + "productive": true +} diff --git a/examples/flaky-test-triage/.loop/state.json b/examples/flaky-test-triage/.loop/state.json new file mode 100644 index 0000000..01da057 --- /dev/null +++ b/examples/flaky-test-triage/.loop/state.json @@ -0,0 +1,14 @@ +{ + "schema": "loop-engineer/state@1", + "project": "flaky-test-triage", + "iteration_id": 2, + "state": "terminal", + "plan_version": 1, + "active_task": "T1", + "best_score": 1.0, + "failure_mode": null, + "pending_approval": null, + "budget_remaining": { "time": "14m", "cost": "0.71usd" }, + "checkpoint_path": "", + "terminal_state": "Succeeded" +} diff --git a/examples/flaky-test-triage/.loop/terminal_state.json b/examples/flaky-test-triage/.loop/terminal_state.json new file mode 100644 index 0000000..fba0379 --- /dev/null +++ b/examples/flaky-test-triage/.loop/terminal_state.json @@ -0,0 +1,14 @@ +{ + "schema": "loop-engineer/terminal@1", + "project": "flaky-test-triage", + "state": "Succeeded", + "iteration_id": 2, + "terminated_at": "2026-07-08T10:31:00Z", + "goal": "Make the scheduler's job-order test deterministic across interpreter runs.", + "criteria_met": { "1": true, "2": true }, + "evidence": [".loop/artifacts/verify-T1.json", ".loop/artifacts/holdout-verdict.json"], + "false_completion": false, + "reason": "Stability score 1.0 under all 5 probe seeds (scripts/verify-full); held-out order-independence property green via a real holdout_gate run. The repair record carries the anchored 0.6 -> 1.0 delta.", + "total_iterations": 2, + "total_repair_attempts": 1 +} diff --git a/examples/flaky-test-triage/README.md b/examples/flaky-test-triage/README.md new file mode 100644 index 0000000..2b05fa8 --- /dev/null +++ b/examples/flaky-test-triage/README.md @@ -0,0 +1,96 @@ +# Worked example — flaky-test-triage + +A second self-contained walk through the **loop-engineer** lifecycle, built to showcase one pillar +the flagship (`examples/coverage-repair/`) only touches in passing: the **structured repair record** +and the **repair-productivity (RP)** metric it feeds. + +> **Scenario:** *A genuinely flaky test — tie-order in a priority sort over an unordered set — is +> triaged and repaired. The order is deterministic within one interpreter but changes across +> `PYTHONHASHSEED` values, so the visible test passes on some seeds and fails on others.* + +## The bug, and why it is really flaky + +`target/jobs.py` orders jobs by priority, highest first. Equal-priority jobs used to be left in +**set-iteration order** — stable within one process, but Python randomizes string hashing per +`PYTHONHASHSEED`, so the tie order (and the test result) changes from run to run. The repair makes +the sort key `(-priority, name)`, so the order is a function of the data, not the interpreter state. + +`target/measure_stability.py` turns that flake into a deterministic probe: it runs the visible test +under 5 fixed `PYTHONHASHSEED` values and reports the passing fraction. This is real, not narrated — +swap the key back to priority alone and the score drops below 1.0. + +## The repair-record pillar + +The single bounded repair pass is recorded at its canonical path, +`.loop/repair/iter-002.json` (`loop-engineer/repair@1`, the 7 canonical fields). Its +`verification_before`/`verification_after` scores are anchored to a same-task red→green pair of +deterministic verify bundles: + +| Artifact | Outcome | score | +|---|---|---| +| `.loop/artifacts/verify-T1-iter1.json` (red) | FAIL | `0.6` — visible test passed under 3/5 probe seeds | +| `.loop/artifacts/verify-T1.json` (green) | PASS | `1.0` — 5/5 probe seeds after the `(priority, name)` key | + +`loop metrics` recomputes `productive` from that `0.6 → 1.0` delta (never trusts the record's own +flag) and anchors it to the two bundles, so the scorecard reports a **non-null RP**: + +``` +repair_productivity = 1.0 # one repair pass, measurably productive +repair_passes = 1 +productive_repairs = 1 +false_completion_rate = 0.0 # the terminal claim's iteration carries only a green bundle +evidence_backed = true # a real held-out gate verdict is on disk +provenance.fcr_methods_agree = true # deterministic cross-join and held-out flag agree +``` + +Every number above is derived from the committed files, not asserted — run the command below to +re-derive it byte-for-byte. + +## The three commands + +```bash +# 1. The contract objects are valid (manifest/state/tasks/terminal/repair): ok == true +python3 -m loop doctor examples/flaky-test-triage + +# 2. Prime-directive score (verify surface invokes the gate; all 7 terminal states in WORKFLOW): +# verdict "strong", score 90 +python3 -m loop inspect examples/flaky-test-triage + +# 3. The FCR/RP scorecard derived from this loop's real .loop/ evidence: +# repair_productivity 1.0, false_completion_rate 0.0, evidence_backed true +python3 -m loop metrics examples/flaky-test-triage +``` + +## Run it yourself + +One entrypoint re-derives the committed held-out verdict from a **live** gate run and checks the +terminal claim against it — no installs, ~1–2s: + +```bash +bash examples/flaky-test-triage/scripts/run-example +``` + +It runs the **real** repo held-out gate `scripts/holdout_gate.py` over the toy target's visible + +holdout checks, writes the verdict to `.loop/artifacts/holdout-verdict.json`, and asserts the +committed `terminal_state.json`'s `false_completion: false` is **backed** by an independent +`Succeeded` verdict — exiting nonzero on any mismatch. `scripts/verify-full` runs the same stability +score + held-out gate as the milestone check. Delete the toy target or the gate wiring and both the +run and `loop metrics` go red. + +## What to notice + +1. **A non-null RP needs anchored evidence, not a flag.** The repair record's `productive: true` is + only counted because a same-task red→green verify-bundle pair (`0.6 → 1.0`) corroborates it — a + fabricated or free-floating number is rejected, not summed. +2. **The flake is deterministic per seed.** `measure_stability.py` makes an intermittent failure + reproducible, so the red bundle's `0.6` is a real measurement, not a story. +3. **Verification is the source of "done."** `terminal_state.json` is `Succeeded` only because the + stability score reached `1.0` and the held-out order-property gate passed — both evidence paths + attached, `false_completion: false`. + +## Where to go next + +- The flagship coverage-repair walkthrough → `examples/coverage-repair/README.md`. +- The schema behind every file here → `reference/repo-os-contract.md`. +- The two first-class metrics (`false-completion-rate` / `repair-productivity`) → + `reference/eval-suite.md`. diff --git a/examples/flaky-test-triage/RUNLOG.md b/examples/flaky-test-triage/RUNLOG.md new file mode 100644 index 0000000..02d2b3a --- /dev/null +++ b/examples/flaky-test-triage/RUNLOG.md @@ -0,0 +1,25 @@ +# RUNLOG.md — flaky-test-triage + +> Human-readable iteration history. Machine state lives in `.loop/state.json`. + +--- + +## Iteration 1 — 2026-07-08T10:08:00Z + +- **active_task:** `T1` — Make next_jobs tie-order deterministic +- **action:** Reproduced the flake: `target/measure_stability.py` runs the visible + test under 5 fixed PYTHONHASHSEED values; the buggy priority-only sort key + passed 3/5. +- **verify:** `scripts/verify-full` → FAIL — stability score 0.6 < 1.0 +- **outcome:** repair_triggered +- **evidence:** `.loop/artifacts/verify-T1-iter1.json` + +## Iteration 2 — 2026-07-08T10:24:00Z + +- **active_task:** `T1` +- **action:** Applied the repair from `.loop/repair/iter-002.json`: sort key + (priority, name) so tie order is a function of the data. +- **verify:** `scripts/verify-full` → PASS — stability score 1.0; held-out order + property green (holdout_gate verdict: Succeeded) +- **outcome:** task_passed +- **evidence:** `.loop/artifacts/verify-T1.json`, `.loop/artifacts/holdout-verdict.json` diff --git a/examples/flaky-test-triage/SPEC.md b/examples/flaky-test-triage/SPEC.md new file mode 100644 index 0000000..25fd023 --- /dev/null +++ b/examples/flaky-test-triage/SPEC.md @@ -0,0 +1,55 @@ +# SPEC.md — flaky-test-triage + +> Intent layer. Describes WHAT success looks like, not HOW to achieve it. +> Scaffolded by `loop-contract` from the ADR. Written once; amended only if the goal changes. + +## Goal + +Make the toy scheduler's job-order test deterministic across interpreter runs. `next_jobs` +orders jobs by priority; equal-priority jobs currently keep set-iteration order, which is stable +within one process but changes across `PYTHONHASHSEED` values — a genuinely flaky test. Make the +tie order a function of the data so the same input always yields the same order. + +## Success Criteria + +Each criterion is independently verifiable; a criterion with no evidence rule is itself a spec gap. + +1. The visible suite passes under all 5 probe `PYTHONHASHSEED` values — stability `score == 1.0` + (`target/measure_stability.py`). +2. Job order is a function of the data, independent of input order — the held-out order-property + probe is green (`target/test_holdout.py`), and the held-out gate certifies `Succeeded`. + +## Constraints + +Things the loop must NOT do, regardless of outcome: + +- Do not modify the public `next_jobs` / `load_jobs` signatures. +- No new third-party dependencies. +- Do not edit tests, fixtures, golden files, or the verify scripts to manufacture a passing gate. + +## Non-goals + +Explicitly out of scope for this loop: + +- Changing the priority ordering itself (higher priority still runs first). +- Adding new job types or scheduling policy. +- Touching any module other than `target/jobs.py` and its tests. + +## Evidence rules + +What counts as proof that each success criterion is met: + +| Criterion | Evidence | Verification command | +|---|---|---| +| 1 — stability `score == 1.0` | passing fraction over 5 fixed probe seeds | `scripts/verify-full` | +| 2 — order-independence | held-out probe passes; held-out gate verdict `Succeeded` | `scripts/verify-full` | + +## Underspecified-criteria rule + +If any success criterion cannot be reduced to a concrete, checkable evidence rule, treat the loop +as `FailedSpecGap` rather than proceeding. Both criteria map to a runnable `scripts/verify-*` gate, +so this loop is well-specified. + +## Risk profile + +`low` — workspace-write only, no external services, fully reversible (one sort key + its tests). diff --git a/examples/flaky-test-triage/TASKS.json b/examples/flaky-test-triage/TASKS.json new file mode 100644 index 0000000..d419f40 --- /dev/null +++ b/examples/flaky-test-triage/TASKS.json @@ -0,0 +1,23 @@ +{ + "schema": "loop-engineer/tasks@1", + "project": "flaky-test-triage", + "tasks": [ + { + "id": "T1", + "title": "Make next_jobs tie-order deterministic (stable sort key, not set-iteration order)", + "status": "done", + "criterion_ref": "1", + "verify": "scripts/verify-full", + "depends_on": [], + "attempts": 2, + "evidence": ".loop/artifacts/verify-T1.json" + } + ], + "metadata": { + "created_at": "2026-07-08T10:02:00Z", + "updated_at": "2026-07-08T10:31:00Z", + "total_tasks": 1, + "completed": 1, + "failed": 0 + } +} diff --git a/examples/flaky-test-triage/WORKFLOW.md b/examples/flaky-test-triage/WORKFLOW.md new file mode 100644 index 0000000..3890217 --- /dev/null +++ b/examples/flaky-test-triage/WORKFLOW.md @@ -0,0 +1,77 @@ +# WORKFLOW.md — flaky-test-triage Loop Policy + +> Stable loop policy. Governs HOW the loop runs, not WHAT it builds. +> Scaffolded by `loop-contract`. Do NOT change during a run without re-entering the `plan` state. + +## Loop + +``` +intake → plan → critique-plan → queue-tasks + → execute-task → verify + → [pass] → next-task or terminal(Succeeded) + → [fail] → repair (cap N=2) → [fixed] → verify + → [cap exceeded] → replan | revert | approve | terminate + → [approval needed] → approval-wait → resume + → [budget exceeded] → terminal(FailedBudget) + → [safety violation] → terminal(FailedSafety) + → [spec gap] → terminal(FailedSpecGap) + → [human abort] → terminal(AbortedByHuman) +``` + +State is externalized to `.loop/state.json` after every transition. +Resume rule: if `.loop/state.json` exists with `terminal_state: null`, skip intake and continue +from `state`. `[[loop-run]]` operates this machine one transition per turn. + +## Approval gates + +Side-effect boundaries that pause for approval: destructive commands, secret access, production +changes. `approval_policy`: `on_side_effects`. This loop is `risk_profile: low` (workspace-write +only), so no gate fired during the run — changing a sort key and re-running probes are not side +effects. Approval gates pause-and-resume from the same `.loop/state.json` checkpoint; they never +spawn a fresh untracked attempt. `plan_then_execute` is `false` (trusted local workspace); set it +`true` for untrusted/web environments. + +## Budgets + +- **Time budget:** `30m` · **Cost budget:** `1.00usd` +- Tracked in `.loop/state.json` as `budget_remaining`. On exhaustion → `FailedBudget` immediately. + +## Repair cap + +- **Max repair attempts per task:** `2` (default). After exceeding: replan / revert / approve / + terminate — never silently retry. Each attempt produces a structured repair record + (`.loop/repair/iter-002.json`). A repair that does not measurably improve the score is churn → + replan. Detected verifier-gaming → hard-terminate `FailedSafety` immediately. + +## Terminal states + +Exactly 7. No other string is a valid terminal state. No silent "completed." + +| State | Fires when | +|---|---| +| `Succeeded` | All success criteria verified with evidence | +| `FailedUnverifiable` | Cannot produce or run verification; evidence missing or contradicting | +| `FailedBlocked` | External dependency, permission, or tool boundary prevents progress | +| `FailedBudget` | `time_budget` or `cost_budget` exhausted before `Succeeded` | +| `FailedSafety` | Safety violation, approval bypass, or verifier-gaming detected | +| `FailedSpecGap` | Success criteria undefined, contradictory, or unverifiable by design | +| `AbortedByHuman` | A human explicitly stopped the run | + +When a terminal state is reached, write `.loop/terminal_state.json` and stop. Never claim +`Succeeded` without verification evidence; unverified completion → `FailedUnverifiable`. + +## Verification + +- `scripts/verify-fast` — quick per-iteration probe (visible suite under one fixed seed). +- `scripts/verify-full` — full gate: stability score over 5 seeds + the REAL repo held-out gate + (`scripts/holdout_gate.py`) over the toy target's visible + holdout checks. Acceptance + verification delegates to `/verify-slice`; this loop builds **no new verifier**. The + deterministic gate is binary and BLOCKING; any rubric judge is advisory only. +- Do not modify tests or verification scripts to make them pass. Passing verification by editing + the verifier is `FailedSafety` + logged as a security failure. + +## Dispatch (model-routing HARD CONTRACT) + +Every dispatched agent names an explicit `model:` — read→`haiku`, reason→`sonnet`, write→`opus`, +orchestrate→main loop. A live run appends receipts to `.loop/receipts/*.jsonl`; this frozen +example ships the contract artifacts, not a receipts trail. diff --git a/examples/flaky-test-triage/scripts/run-example b/examples/flaky-test-triage/scripts/run-example new file mode 100644 index 0000000..83dabc1 --- /dev/null +++ b/examples/flaky-test-triage/scripts/run-example @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# run-example — re-derive the committed holdout verdict from a LIVE gate run, +# then check the committed terminal claim against it. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +REPO="$(cd "$EX/../.." && pwd)" + +python3 -B "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target" \ + > "$EX/.loop/artifacts/holdout-verdict.json" + +python3 - "$EX" <<'PY' +import json, sys +from pathlib import Path + +example = Path(sys.argv[1]) +verdict = json.loads((example / ".loop/artifacts/holdout-verdict.json").read_text()) +terminal = json.loads((example / ".loop/terminal_state.json").read_text()) +assert verdict["verdict"] == "Succeeded", verdict +assert verdict["false_completion"] is False, verdict +assert terminal["false_completion"] is False +print("terminal claim BACKED by an independent holdout_gate run (verdict: Succeeded)") +PY diff --git a/examples/flaky-test-triage/scripts/verify-fast b/examples/flaky-test-triage/scripts/verify-fast new file mode 100644 index 0000000..9dde75c --- /dev/null +++ b/examples/flaky-test-triage/scripts/verify-fast @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# verify-fast — criterion 1 quick probe: the visible suite under one fixed seed. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +( cd "$EX/target" && PYTHONHASHSEED=3 python3 -B -m pytest -q -p no:cacheprovider test_visible.py ) +echo "verify-fast: PASS (visible test green under seed 3)" diff --git a/examples/flaky-test-triage/scripts/verify-full b/examples/flaky-test-triage/scripts/verify-full new file mode 100644 index 0000000..6333521 --- /dev/null +++ b/examples/flaky-test-triage/scripts/verify-full @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# verify-full — criterion 1 (stability score over 5 seeds) + false-completion +# defense: the REAL repo held-out gate over the toy target's visible + holdout. +set -euo pipefail +EX="$(cd "$(dirname "$0")/.." && pwd)" +REPO="$(cd "$EX/../.." && pwd)" + +bash "$EX/scripts/verify-fast" + +echo "== criterion 1: visible stability score == 1.0 over 5 probe seeds ==" +( cd "$EX/target" && python3 -B measure_stability.py ) + +echo "== false-completion defense: held-out gate over visible + holdout ==" +python3 "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target" +echo "verify-full: PASS (criteria 1+2 verified; holdout gate green)" diff --git a/examples/flaky-test-triage/target/jobs.py b/examples/flaky-test-triage/target/jobs.py new file mode 100644 index 0000000..1ff65d3 --- /dev/null +++ b/examples/flaky-test-triage/target/jobs.py @@ -0,0 +1,17 @@ +"""Toy job scheduler: choose the next jobs to run, highest priority first.""" + + +def load_jobs(): + """Job records as an unordered set — insertion order is not meaningful.""" + return {("archive", 2), ("compact", 2), ("backup", 1)} + + +def next_jobs(jobs): + """Job names ordered by priority (highest first), ties broken by name. + + The T1 repair: the sort key used to be priority alone, which left the + order of equal-priority jobs to set-iteration order — stable within one + process, different across PYTHONHASHSEED values. The (priority, name) + key makes the order a function of the data, not the interpreter state. + """ + return [name for name, _prio in sorted(jobs, key=lambda j: (-j[1], j[0]))] diff --git a/examples/flaky-test-triage/target/manifest.json b/examples/flaky-test-triage/target/manifest.json new file mode 100644 index 0000000..8b1d49c --- /dev/null +++ b/examples/flaky-test-triage/target/manifest.json @@ -0,0 +1,8 @@ +{ + "visible": [ + { "id": "visible-stability", "cmd": "python3 -B measure_stability.py" } + ], + "holdout": [ + { "id": "holdout-order-property", "cmd": "python3 -B -m pytest -q -p no:cacheprovider test_holdout.py" } + ] +} diff --git a/examples/flaky-test-triage/target/measure_stability.py b/examples/flaky-test-triage/target/measure_stability.py new file mode 100644 index 0000000..9e8deff --- /dev/null +++ b/examples/flaky-test-triage/target/measure_stability.py @@ -0,0 +1,32 @@ +"""Deterministic flakiness probe: run the visible suite under 5 fixed +PYTHONHASHSEED values; the score is the passing fraction. Before the T1 +repair the tie order tracked set-iteration order, so some seeds failed +(committed red bundle: score 0.4); after it, 5/5 pass every time.""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +SEEDS = ("0", "1", "2", "3", "4") + + +def main() -> int: + passed = 0 + for seed in SEEDS: + env = dict(os.environ, PYTHONHASHSEED=seed) + proc = subprocess.run( + [sys.executable, "-B", "-m", "pytest", "-q", "-p", "no:cacheprovider", + str(HERE / "test_visible.py")], + env=env, capture_output=True, text=True, cwd=HERE, + ) + passed += proc.returncode == 0 + score = passed / len(SEEDS) + print(json.dumps({"seeds": len(SEEDS), "passed": passed, "score": score})) + return 0 if passed == len(SEEDS) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/flaky-test-triage/target/test_holdout.py b/examples/flaky-test-triage/target/test_holdout.py new file mode 100644 index 0000000..8c3d14e --- /dev/null +++ b/examples/flaky-test-triage/target/test_holdout.py @@ -0,0 +1,15 @@ +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from jobs import next_jobs + + +def test_order_is_independent_of_input_order(): + base = [("archive", 2), ("compact", 2), ("backup", 1), ("prune", 3)] + rng = random.Random(7) + for _ in range(20): + shuffled = list(base) + rng.shuffle(shuffled) + assert next_jobs(shuffled) == ["prune", "archive", "compact", "backup"] diff --git a/examples/flaky-test-triage/target/test_visible.py b/examples/flaky-test-triage/target/test_visible.py new file mode 100644 index 0000000..4107f5d --- /dev/null +++ b/examples/flaky-test-triage/target/test_visible.py @@ -0,0 +1,9 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from jobs import load_jobs, next_jobs + + +def test_equal_priority_jobs_run_in_stable_order(): + assert next_jobs(load_jobs()) == ["archive", "compact", "backup"] diff --git a/scripts/test_flaky_example.py b/scripts/test_flaky_example.py new file mode 100644 index 0000000..74ea762 --- /dev/null +++ b/scripts/test_flaky_example.py @@ -0,0 +1,55 @@ +"""ST4 acceptance: the 2nd runnable example is doctor-clean and gate-backed, +and showcases the repair-record pillar — `loop metrics` derives a non-null RP +from a same-task red->green anchored repair. run-example re-derives the +committed gate verdict live from a foreign cwd.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parent.parent +_EXAMPLE = _REPO / "examples" / "flaky-test-triage" + + +def _cli(cmd: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", cmd, str(_EXAMPLE)], + cwd=_REPO, capture_output=True, text=True, + ) + + +def test_doctor_clean(): + proc = _cli("doctor") + assert proc.returncode == 0, proc.stdout + proc.stderr + assert json.loads(proc.stdout)["ok"] is True + + +def test_metrics_derives_non_null_rp_and_clean_fcr(): + proc = _cli("metrics") + assert proc.returncode == 0, proc.stdout + proc.stderr + card = json.loads(proc.stdout) + assert card["repair_productivity"] == 1.0 + assert card["repair_passes"] == 1 + assert card["productive_repairs"] == 1 + assert card["false_completion_rate"] == 0.0 + assert card["evidence_backed"] is True + prov = card["provenance"] + assert prov["fcr_methods_agree"] is True + assert prov["rejected_records"] == [] + assert prov["unanchored_records"] == [] + assert prov["unmatched_verify"] == [] + + +def test_run_example_reproduces_the_gate_verdict_from_foreign_cwd(tmp_path): + proc = subprocess.run( + ["bash", str(_EXAMPLE / "scripts" / "run-example")], + cwd=tmp_path, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert "BACKED by an independent" in proc.stdout + verdict = json.loads((_EXAMPLE / ".loop" / "artifacts" / "holdout-verdict.json").read_text()) + assert verdict["verdict"] == "Succeeded" + assert verdict["false_completion"] is False From 0bf03276271d46393930d32a4c3af5acafdffc22 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 22:00:22 -0400 Subject: [PATCH 05/10] =?UTF-8?q?fix(st4):=20flaky-triage=20=E2=80=94=20st?= =?UTF-8?q?ray=200.4=20docstring=20vs=20committed=200.6=20red=20bundle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/flaky-test-triage/target/measure_stability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/flaky-test-triage/target/measure_stability.py b/examples/flaky-test-triage/target/measure_stability.py index 9e8deff..83b3890 100644 --- a/examples/flaky-test-triage/target/measure_stability.py +++ b/examples/flaky-test-triage/target/measure_stability.py @@ -1,7 +1,7 @@ """Deterministic flakiness probe: run the visible suite under 5 fixed PYTHONHASHSEED values; the score is the passing fraction. Before the T1 repair the tie order tracked set-iteration order, so some seeds failed -(committed red bundle: score 0.4); after it, 5/5 pass every time.""" +(committed red bundle: score 0.6); after it, 5/5 pass every time.""" import json import os From b03f4caab9e1dbd755accb850277f280d3f597c4 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 22:14:00 -0400 Subject: [PATCH 06/10] =?UTF-8?q?docs(st4):=20contributor=20funnel=20?= =?UTF-8?q?=E2=80=94=20seven=20gate-backed=20issue=20drafts=20+=20CONTRIBU?= =?UTF-8?q?TING=20start-here=20section?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONTRIBUTING.md | 19 ++++++ .../01-good-first-qw9-trigger-phrases.md | 45 ++++++++++++++ .../02-good-first-qw10-self-eval-labels.md | 39 ++++++++++++ .../03-good-first-emit-metrics-vocabulary.md | 44 +++++++++++++ .../issues/04-help-wanted-openhands-recipe.md | 42 +++++++++++++ .../issues/05-help-wanted-ruflo-recipe.md | 39 ++++++++++++ .../issues/06-help-wanted-gap-reports.md | 42 +++++++++++++ ...07-good-first-emit-scaffold-runlog-seed.md | 61 +++++++++++++++++++ 8 files changed, 331 insertions(+) create mode 100644 docs/contributing/issues/01-good-first-qw9-trigger-phrases.md create mode 100644 docs/contributing/issues/02-good-first-qw10-self-eval-labels.md create mode 100644 docs/contributing/issues/03-good-first-emit-metrics-vocabulary.md create mode 100644 docs/contributing/issues/04-help-wanted-openhands-recipe.md create mode 100644 docs/contributing/issues/05-help-wanted-ruflo-recipe.md create mode 100644 docs/contributing/issues/06-help-wanted-gap-reports.md create mode 100644 docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b843a5..15fc7cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,25 @@ python3 -m loop inspect examples/coverage-repair If you don't have the deps, prefix with `uv run --with pyyaml --with pytest`. +## Start here — the contributor funnel + +Every open starter issue names **the gate that proves the fix** — a +deterministic command that is red before your change and green after. That is +the whole review bar (see the ground rule above). + +- [`good first issue`](https://github.com/SollanSystems/loop-engineer/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + — small, bounded, gate-verifiable fixes. +- [`help wanted`](https://github.com/SollanSystems/loop-engineer/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) + — integration recipes (OpenHands, ruflo — designs already written in + `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md`) and + foreign-harness gap reports (`docs/gap-reports/`). + +The contribution target for anything that emits or consumes contract +artifacts is the standard: `reference/repo-os-contract.md` — a harness that +satisfies the §14 conformance checklist (A1–E1) may claim it emits a +Loop-Engineer-conformant contract v1. Drafts for the seeded issues live in +`docs/contributing/issues/` and are filed on GitHub at release time. + ## Repository layout | Path | What lives here | diff --git a/docs/contributing/issues/01-good-first-qw9-trigger-phrases.md b/docs/contributing/issues/01-good-first-qw9-trigger-phrases.md new file mode 100644 index 0000000..0ee5ed7 --- /dev/null +++ b/docs/contributing/issues/01-good-first-qw9-trigger-phrases.md @@ -0,0 +1,45 @@ + + + +# Trigger-phrase disambiguation batch (3 LOW fixes) + +Three small `SKILL.md` frontmatter edits that sharpen router resolution. Each is +independently checkable and none changes behavior — only the trigger prose. + +## Problem + +Three trigger-phrase weaknesses, all present at this commit: + +- **Shared bare "grade" verb.** `skills/loop-evals/SKILL.md` anchors *"or grade a + long-running agentic run"* and `skills/loop-inspector/SKILL.md` anchors *"grade a + superpowers / ruflo / .loop harness"* — both hang the same verb on different + objects, so a bare "grade this" query has no clean winner. +- **`loop-evals` verbosity outlier.** Its `description` scalar runs well past the + sibling band (~736 chars vs a ~400–510-char peer range), because capability prose + (the 7-layer suite, deterministic-then-rubric, the regression harness) lives in the + frontmatter instead of the body. +- **`loop-run` weak first example.** `skills/loop-run/SKILL.md` opens its example + list with bare `'run the loop'` — the only one of its examples lacking a + qualifier. + +## Proposal + +- Make the noun part of each "grade" phrase: evals → *"grade a run's outcome against + its SPEC"*; inspector → *"grade this harness/contract's readiness"*. +- Trim `loop-evals`' `description` into the ~400–510-char sibling band by moving the + capability-summary prose into the skill body, leaving trigger phrases + a one-line + hook in frontmatter. +- Qualify `loop-run`'s opening example: *"run the agent loop"* (or *"run this loop's + state machine"*). + +Keep every `description` a *quoted* YAML scalar (the suite quotes all of them). + +## The gate that proves the fix + +```bash +python3 scripts/validate_frontmatter.py # green +python3 scripts/self_eval.py # green +``` + +`loop-evals`' frontmatter length should land back in the sibling band; the two +gates above are the whole review bar. diff --git a/docs/contributing/issues/02-good-first-qw10-self-eval-labels.md b/docs/contributing/issues/02-good-first-qw10-self-eval-labels.md new file mode 100644 index 0000000..7a41cb1 --- /dev/null +++ b/docs/contributing/issues/02-good-first-qw10-self-eval-labels.md @@ -0,0 +1,39 @@ + + + +# Label self_eval terminal/repair/eval checks honestly as doc-completeness + +A naming/comment clarification (no behavioral change required) that keeps the gate +honest about its own scope. + +## Problem + +Three `scripts/self_eval.py` checks are substring-presence scans over a single +`SKILL.md` each, not behavioral enforcement: + +- `check_terminal_states` (`scripts/self_eval.py:185`) — `missing = [s for s in + facts["terminal_states"] if s not in text]` over `skills/loop-run/SKILL.md`. +- `check_repair_fields` (`scripts/self_eval.py:193`) — same shape over + `skills/loop-repair/SKILL.md`. +- `check_eval_layers_and_metrics` (`scripts/self_eval.py:206`) — normalized + substring presence over `skills/loop-evals/SKILL.md`. + +Each passes as long as the canonical words *appear in the prose*. Presenting them as +"the hard pass/fail gate" risks a reader mistaking documentation-completeness for +runtime-correctness enforcement — gaming requires only listing the canonical words. + +## Proposal + +Rename and/or comment the three checks as **documentation-completeness** checks (not +behavioral enforcement), and say so where `self_eval` is described as a gate — in +`CONTRIBUTING.md` (the "Ground rule" / self_eval mention) and in the README's +structural-check list. No behavioral change is required if the checks are +intentional; this is a scope-honesty clarification consistent with the suite's own +posture. + +## The gate that proves the fix + +```bash +python3 scripts/self_eval.py # green (13 structural invariants) +python3 -m pytest -q scripts/test_docs_claims.py # README accuracy assertions green +``` diff --git a/docs/contributing/issues/03-good-first-emit-metrics-vocabulary.md b/docs/contributing/issues/03-good-first-emit-metrics-vocabulary.md new file mode 100644 index 0000000..e8f4f0a --- /dev/null +++ b/docs/contributing/issues/03-good-first-emit-metrics-vocabulary.md @@ -0,0 +1,44 @@ + + + +# Reconcile emit's iteration-outcome vocabulary with metrics' recognized tokens + +The writer and the metrics reader disagree on the outcome vocabulary, so a RUNLOG +written entirely through the sanctioned writer can still look "dirty" to `loop +metrics`. + +## Problem + +`loop/emit.py` `_ITERATION_OUTCOMES` (`loop/emit.py:26`) accepts `approval_requested` +and `replanned`, but `scripts/metrics.py` `_KNOWN_OUTCOME_TOKENS` +(`scripts/metrics.py:101`, built from `_SUCCESS_OUTCOME_TOKENS` + +`_HONEST_RED_OUTCOME_TOKENS`) recognizes neither. So an iteration appended via +`emit.append_iteration(..., outcome="approval_requested")` — a fully valid write — +surfaces under `provenance.unrecognized_outcomes`. + +Checkable at this commit: + +```python +import sys; sys.path.insert(0, "scripts"); import metrics +from loop.emit import _ITERATION_OUTCOMES +print([o for o in _ITERATION_OUTCOMES if o not in metrics._KNOWN_OUTCOME_TOKENS]) +# -> ['approval_requested', 'replanned'] +``` + +## Proposal + +Decide the canonical vocabulary and align the two ends: + +- either add `approval_requested` and `replanned` to metrics' honest-red set + (`scripts/metrics.py` `_HONEST_RED_OUTCOME_TOKENS`) — they are known, non-success + outcomes, so they belong there and should not read as "unrecognized synonyms"; +- or narrow `emit`'s accepted set to the tokens metrics already knows. + +Add a round-trip regression test: `emit.append_iteration` writing every allowed +outcome, then `compute_metrics(...)` reports `provenance.unrecognized_outcomes == []`. + +## The gate that proves the fix + +```bash +python3 -m pytest scripts/test_metrics.py scripts/test_emit.py # green +``` diff --git a/docs/contributing/issues/04-help-wanted-openhands-recipe.md b/docs/contributing/issues/04-help-wanted-openhands-recipe.md new file mode 100644 index 0000000..172ae39 --- /dev/null +++ b/docs/contributing/issues/04-help-wanted-openhands-recipe.md @@ -0,0 +1,42 @@ + + + +# Integration recipe: OpenHands run → FCR gate + +An integration recipe that layers the loop-contract's false-completion gate over an +OpenHands run. **Composes, doesn't compete:** OpenHands is the EXECUTE tier — it +writes, runs, and tests code in a sandbox; "done" is still the agent stopping. Loop +Engineer wraps that exit in a typed terminal + held-out gate. This adds a layer, it +does not replace the runtime. + +## Problem / opportunity + +The design is already written — `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md` +§5.3 — but no runnable recipe ships yet. The shipped LangGraph and Temporal recipes +are the template to follow: + +- the adapter seam: `loop/integrations.py` (`EngineOutcome` → `to_terminal_state`); +- an env-guarded end-to-end example under `examples/` (skips cleanly when the engine + isn't installed, like `examples/langgraph-emit/` and `examples/temporal-certify/`); +- a CI job that exercises it. + +The specialization from §5.3: OpenHands' `AgentStuckError` / max-iteration stop maps +to `FailedBudget`; a sandbox that touched a holdout/answer-key path (HIGH anti-cheat +finding) maps to `FailedUnverifiable` — the exact "the runtime ran tests but the +agent peeked" case OpenHands can't itself catch. + +## Proposal + +Follow the `docs/integrations/langgraph.md` recipe end-to-end for OpenHands: read the +run's trajectory as the anti-cheat trajectory input, run the holdout split, and +project the engine outcome through `to_terminal_state` into a typed terminal. + +## The gate that proves the fix + +```bash +python3 -m loop doctor # doctor round-trip is clean +``` + +Plus a pinned false-completion invariant test: visible-green / holdout-red must +project to `FailedUnverifiable` with `false_completion: true`, and **never** +`Succeeded`. diff --git a/docs/contributing/issues/05-help-wanted-ruflo-recipe.md b/docs/contributing/issues/05-help-wanted-ruflo-recipe.md new file mode 100644 index 0000000..fbe5da8 --- /dev/null +++ b/docs/contributing/issues/05-help-wanted-ruflo-recipe.md @@ -0,0 +1,39 @@ + + + +# Integration recipe: ruflo swarm → acceptance gate + +An integration recipe that layers a single acceptance gate over a ruflo swarm. +**Composes, doesn't compete:** ruflo is the ORCHESTRATE tier — a multi-agent swarm +whose terminal is "the coordinator decided the objective is met," a self-report +across N agents. Loop Engineer adds one acceptance gate the swarm must pass *as a +whole*. This adds a layer, it does not replace the swarm. + +## Problem / opportunity + +The design is written — `docs/superpowers/specs/2026-06-30-st3-integration-adapters.md` +§5.4 — but no runnable recipe ships yet. Same shape as the OpenHands recipe (issue +04): follow the shipped LangGraph/Temporal recipes as the template (`loop/integrations.py` +adapter, an env-guarded end-to-end example, a CI job). + +The seam is the swarm's terminal hook: **no individual agent may declare the swarm +done** — the acceptance gate does. Register the gate as the swarm's terminal hook +(ruflo exposes hooks / an MCP coordination server), read the merged diff and agent +trails as the anti-cheat inputs, run the holdout split, and project through +`to_terminal_state`. + +## Proposal + +Follow the `docs/integrations/langgraph.md` recipe end-to-end for a ruflo swarm, +wiring the gate as the terminal hook so the swarm's collective "done" is decided by +the held-out gate, not by any agent's self-report. + +## The gate that proves the fix + +```bash +python3 -m loop doctor # doctor round-trip is clean +``` + +Plus a pinned false-completion invariant test: visible-green / holdout-red must +project to `FailedUnverifiable` with `false_completion: true`, and **never** +`Succeeded`. diff --git a/docs/contributing/issues/06-help-wanted-gap-reports.md b/docs/contributing/issues/06-help-wanted-gap-reports.md new file mode 100644 index 0000000..7d42231 --- /dev/null +++ b/docs/contributing/issues/06-help-wanted-gap-reports.md @@ -0,0 +1,42 @@ + + + +# Foreign-harness gap reports — the inspect scoreboard pipeline + +Contribute a read-only `loop inspect` gap report for another public harness layout — +the second entry in an "inspect N public harnesses" scoreboard. + +## Problem / opportunity + +`docs/gap-reports/superpowers.md` is the template and the first entry. **Composes, +doesn't compete:** a gap report reads a foreign layout read-only and names the gaps a +loop-contract would *close* at the finish line — it is not a criticism of the foreign +harness or of the (fictional) work in the fixture. `inspect` labels a foreign layout +`advisory: true`; a foreign layout is read for gaps, never graded as a failing +contract. + +The template's provenance rules are load-bearing: + +- every claim is checkable against **one vendored, sanitized fixture** checked into + `examples/` — no version-general claims about the foreign harness itself; +- the report carries the §14 A1–E1 conformance table (condensed from + `reference/repo-os-contract.md` §14), read against that fixture; +- complement framing throughout ("composes, doesn't compete"). + +## Proposal + +For another public harness layout: vendor a fictional, sanitized fixture under +`examples/-run/`, then author `docs/gap-reports/.md` following +`docs/gap-reports/superpowers.md` — the provenance blockquote, the §14 A1–E1 table, +the `inspect` reading, and "what emitting the contract would add," with every factual +claim restricted to the fixture. + +## The gate that proves the fix + +```bash +python3 -m loop inspect examples/-run # produces a scored report +``` + +The exit criteria follow the `scripts/test_foreign_inspect.py` patterns (scored, +`foreign_layout` labeled, `advisory: true`), and the report follows the template's +provenance rules. diff --git a/docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md b/docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md new file mode 100644 index 0000000..bb08705 --- /dev/null +++ b/docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md @@ -0,0 +1,61 @@ + + + +# emit.open_contract seeds a metrics-dirty RUNLOG placeholder + +A fresh scaffold is born "metrics-dirty," which forces every integration recipe to +work around it. Fix the root so the workaround can be retired. + +## Problem + +`loop.emit.open_contract` scaffolds a `RUNLOG.md` from `templates/RUNLOG.md.tmpl`, +whose outcome block (`templates/RUNLOG.md.tmpl:27`) is +`` `{{ITERATION_OUTCOME}}` — one of: `task_passed`, … ``. `loop/scaffold.py` +`_substitutions` has no `ITERATION_OUTCOME` key, so `_fill` renders the unknown token +as the literal `REPLACE` (`loop/scaffold.py:95`). `scripts/metrics.py` then parses +`REPLACE` as an unrecognized outcome token, so a just-scaffolded loop is +metrics-dirty until real iterations land. + +Checkable at this commit: + +```python +import sys, tempfile, os +from loop import emit +sys.path.insert(0, "scripts"); import metrics +ws = os.path.join(tempfile.mkdtemp(), "scaffold_check") +emit.open_contract(ws) +print(metrics.compute_metrics(ws)["provenance"]["unrecognized_outcomes"]) +# -> ['replace'] +``` + +Because of this, both engine recipes reset `/RUNLOG.md` to emit's fresh header +right after `open_contract`: `examples/langgraph-emit/graph_example.py:105` and +`examples/temporal-certify/workflow_example.py:159`. That inline reset is a +documented carve-out in the plan's Global Constraints +(`docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md`, "Adjudicated +carve-out"). + +## Proposal + +Make the scaffold not seed a metrics-flagged placeholder. Options, pick one: + +- a first-class affordance — `emit.open_contract(seed_runlog=False)` or + `emit.reset_runlog(ws)` — so a caller that scores from iteration 0 opts out of the + placeholder; or +- render the placeholder outcome as a value `metrics` recognizes (or omit the seeded + iteration block entirely) so a fresh scaffold is metrics-clean by construction. + +The reset only ever strips a placeholder — it never fabricates state — so the +affordance must preserve that (no synthetic completed iteration). + +## The gate that proves the fix / acceptance + +- A freshly-scaffolded loop reports `provenance.unrecognized_outcomes == []` (the + snippet above returns `[]`). +- Both recipes drop their inline `RUNLOG.md` resets and still pass their gates. +- The plan's Global-Constraints carve-out text can be retired. + +```bash +python3 -m pytest scripts/test_metrics.py scripts/test_emit.py # green +python3 -m loop doctor # recipe still clean after the inline reset is removed +``` From f9a4e52c3cc7dd9e6a90c54af60bcd9e70b27dc9 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Wed, 8 Jul 2026 22:22:27 -0400 Subject: [PATCH 07/10] =?UTF-8?q?fix(st4):=20relabel=20draft=2007=20help-w?= =?UTF-8?q?anted=20=E2=80=94=20scope=20spans=20emit/scaffold/templates/rec?= =?UTF-8?q?ipes,=20not=20a=20good=20first=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...nlog-seed.md => 07-help-wanted-emit-scaffold-runlog-seed.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/contributing/issues/{07-good-first-emit-scaffold-runlog-seed.md => 07-help-wanted-emit-scaffold-runlog-seed.md} (98%) diff --git a/docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md b/docs/contributing/issues/07-help-wanted-emit-scaffold-runlog-seed.md similarity index 98% rename from docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md rename to docs/contributing/issues/07-help-wanted-emit-scaffold-runlog-seed.md index bb08705..94b9f5a 100644 --- a/docs/contributing/issues/07-good-first-emit-scaffold-runlog-seed.md +++ b/docs/contributing/issues/07-help-wanted-emit-scaffold-runlog-seed.md @@ -1,5 +1,5 @@ - + # emit.open_contract seeds a metrics-dirty RUNLOG placeholder From 663a1f52ae68ea3c01a8e793d2136af345b2be5d Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Thu, 9 Jul 2026 05:25:20 -0400 Subject: [PATCH 08/10] =?UTF-8?q?chore:=20untrack=20CLAUDE.md=20=E2=80=94?= =?UTF-8?q?=20local=20cold-start=20doc,=20per=20its=20own=20do-not-commit?= =?UTF-8?q?=20note=20and=20PR-A=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 ++ CLAUDE.md | 125 ----------------------------------------------------- 2 files changed, 3 insertions(+), 125 deletions(-) delete mode 100644 CLAUDE.md diff --git a/.gitignore b/.gitignore index 50d23a0..c060a32 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ Thumbs.db # Secrets — never commit .env *.local + +# Local cold-start context — deliberately untracked (see its own do-not-commit note) +/CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 0b4aa92..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,125 +0,0 @@ -# CLAUDE.md — loop-engineer - -Read this first. It replaces the "continue where we left off" cold-start that has been -booting this repo for 9+ days. Nested git repo (`SollanSystems/loop-engineer`, MIT) — -do NOT commit CLAUDE.md or treat it as plugin content. - -## What this is - -A Claude-Code plugin that designs, launches, verifies, repairs, and improves **agent -loops**. It is an **architect + operator**, not a task-doer: it stands up the operating -contract for a long-running agentic-coding run, gates it against false completion, and -mines its own history for compounding improvement. - -Two shipping surfaces from one repo: -- **Plugin** — hub-and-spoke skill suite (router + 8 spokes) invoked inside Claude Code. -- **Portable Loop Contract Core** — a pure-stdlib Python package `loop` + CLI - (`loop` / `loop-engineer`, also `uvx loop-engineer`) that validates and inspects - repo-native operating contracts on any stack. `pip install -e .` pulls **zero** runtime - deps; two optional extras enrich validation: `yaml` (PyYAML) and `schemas` (jsonschema). - -## Skill architecture (9 skills = router + 8 spokes) - -| Skill | Role | -|---|---| -| `loop-engineer` | Router — dispatches broad agent-loop intent to the right spoke. | -| `loop-architect` | The brain — classify the task, choose architecture + Claude-Code realization, emit a structured ADR (architecture, loop patterns, risk profile, terminal-state plan, next spokes). | -| `loop-contract` | Scaffold the repo-OS contract (SPEC / WORKFLOW / TASKS.json / RUNLOG / `.loop/state.json` + `verify-*` skeletons) from an ADR, then run the pre-execution reflection. | -| `loop-run` | The operator — run/resume the state machine one bounded transition at a time; every run ends in exactly one of the 7 terminal states. | -| `loop-repair` | Patch-and-repair loop — classify the failure mode, make the smallest bounded repair, write a 7-field repair record, enforce a max-attempt cap; refuses to widen scope or edit tests to fake a pass. | -| `loop-evals` | Design the eval harness — the 7-layer suite, FCR + repair-productivity as first-class metrics, deterministic-first-then-rubric, judge calibration. Delegates the deterministic gate to the contract's own `verify-*` scripts. | -| `loop-flywheel` | Turn run history (RUNLOG / traces / receipts) into new eval cases; drive baseline→harden→regression→freeze; compact memory. | -| `loop-inspector` | Read-only score of a **foreign** loop against the prime-directive checklist + 7 terminal states → a scored gap report (advisory heuristic, not a gate). | -| `loop-runtime-monitor` | Watch a running loop from outside — detect stall / repair-churn / budget-overrun and surface one intervention. | - -## Version + state - -- **Current release: v0.7.0** (`.claude-plugin/plugin.json` + `pyproject.toml`), main at `4647820`. -- **Active branch: `feat/v0.8.0-composes-the-field`** (v0.8.0 NOT yet released — release cut - is the last commit of PR-B). Working tree has in-progress Temporal recipe files (Task 3). -- **Shipped through v0.7.0:** - - ST2 "Portable standard" — `reference/repo-os-contract.md` promoted to the normative, - versioned, tool-agnostic spec (PR #31). - - External-review credibility patch set **PRs #27–#30** (10 findings): empty-evidence - terminal now fails doctor; terminal write-once + atomic writes; strict UTF-8 + declared - ledger discovery; strict-by-install on the Action + pre-commit hook; verify-script - existence checks; **inspector honesty** (#30) — keyword-stuffed fakes no longer score - 100/strong, `loop inspect` documented as advisory while `loop doctor` is the hard gate. - - Adoption slices (PRs #19–#23): PyPI wheel substrate, `loop/emit.py` writer API + - LangGraph recipe, `hooks/stop_firewall.py` Stop-hook false-completion firewall, - composite `action.yml` + `.pre-commit-hooks.yaml`. -- **v0.8.0 "composes-the-field" (in progress):** ST3 integration adapters - (`loop/integrations.py` EngineOutcome→terminal projection, LangGraph recipe upgraded to - gate+adapter+metrics-clean, Temporal recipe) + ST4 contributor funnel (foreign-harness - inspect adapter `loop/foreign.py`, gap report, `flaky-test-triage` example, contributor - issue drafts + CONTRIBUTING). Resume ledger: `.superpowers/sdd/` (progress.md + briefs). -- **Test baseline: 395 tests** collected on the v0.8.0 branch (372/10 at v0.7.0; ~10 skip - in structural-fallback mode without jsonschema). -- **Dogfood:** the suite has been run on its own v1.0 launch (self-hosted contract at - `roadmap/launch/`) and on `examples/coverage-repair` (doctor-clean, inspect 90/strong). - -## Load-bearing invariants (enforced by `scripts/self_eval.py` vs `evals/cases/structural.json`) - -- **7 canonical terminal states:** `Succeeded`, `FailedUnverifiable`, `FailedBlocked`, - `FailedBudget`, `FailedSafety`, `FailedSpecGap`, `AbortedByHuman`. -- **7-field repair record:** `failure_mode`, `hypothesis`, `repair_action`, - `verification_before`, `verification_after`, `remaining_delta`, `productive`. -- **7-layer eval suite:** deterministic-correctness, artifact-quality, human-calibration, - loop-behavior, security/governance, regression-resistance, cost/efficiency. -- **2 first-class metrics:** `false-completion-rate` (FCR), `repair-productivity` (RP). -- Also pinned: 6-item `failure_mode_taxonomy`, `repair_cap_default` 2, `rubric_target_mean` - 9.5 (advisory), 9 skill names, 8 reference filenames, 14 template filenames, MIT license. -- **When the suite changes** (new skill/template/reference/terminal state), update - `evals/cases/structural.json` — the self-eval checks compare live repo state against it. - -## Verify commands (run from the repo root; this env has no system pytest — use `uv run`) - -```bash -uv run --with pyyaml python3 -B scripts/validate_frontmatter.py # 9 SKILL.md frontmatter blocks -uv run --with pyyaml python3 -B scripts/self_eval.py # structural invariants (self-locates root) -uv run --with pyyaml --with jsonschema --with pytest python3 -B -m pytest -q -p no:cacheprovider scripts # full suite -``` - -CLI subcommands (`python3 -m loop ` — commands: `scaffold doctor validate verify -inspect metrics`): `loop doctor` is the **hard gate**, `loop inspect .` is the **advisory -scorecard**. CI (`.github/workflows/ci.yml`) installs `pyyaml pytest jsonschema` and runs -`python -B -m pytest -q -p no:cacheprovider scripts`; without jsonschema the core falls back -to structural hand-checks (a deliberate design, not a bug). - -## Install / refresh - -Installed **user-scope** from the local marketplace `loop-engineer-local`. The plugin-cache -copy is a **static COPY** (nested under `cache//loop-engineer//`), NOT a symlink — -it goes **stale after any post-install commit**. Refresh with: - -```bash -git -C /mnt/c/Dev/projects/loop-engineer archive HEAD | tar -x -C -diff -rq /mnt/c/Dev/projects/loop-engineer # verify -``` - -Then **restart Claude Code** to reload the skills. - -## Roadmap / open work - -- **Human gates (blocking PyPI):** register the PyPI pending trusted publisher - (project `loop-engineer`, owner `SollanSystems`, workflow `publish.yml`, environment - `pypi`), then `git tag v0.7.0 4647820 && git push origin v0.7.0`; verify the funnel - `uvx loop-engineer@0.7.0 inspect .` from a scratch dir. -- **Finish v0.8.0:** Temporal recipe (Task 3) → ship PR-A → PR-B (foreign adapter, gap - report, flaky example, contributor funnel, 0.8.0 release cut) → file the 6 contributor - issues at merge. -- **Then:** "inspect N public harnesses" scoreboard post; Show HN launch (human-gated). -- **Positioning (verified):** never claim to coin "loop engineering" — cobusgreyling's - `loop-engineering` (~4.6k stars) owns the term. Own **"false completion"** / - **"proof-of-done contract"** instead. - -## Repo conventions - -- Specs → `docs/superpowers/specs/` (dated); plans → `docs/superpowers/plans/` (dated). -- Session narratives → `memory/session-summaries/YYYY-MM-DD-*.md` (memory/ is gitignored). -- **Gitignored workbench/telemetry** (not plugin content): `.loop/`, `review/`, `roadmap/`, - `.claude/`, `.gsd/`, `memory/`, `.tmp/`. CI runs on a fresh checkout where these are - absent — never point a CI/dogfood job at the live gitignored `.loop/`; target a tracked - `examples/*` contract instead. -- Env quirks: the Bash deny-list blocks `rm`, bare `cd`, `VAR=` assignments, `timeout`, - `printf`, `source` — use `git -C `, literal absolute paths, and `bash -c`. From e78d5e3d71bb218dc701d9530fd27d291ccb80c3 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Thu, 9 Jul 2026 20:28:11 -0400 Subject: [PATCH 09/10] =?UTF-8?q?fix(st4):=20run-example=20pytest=20depend?= =?UTF-8?q?ency=20=E2=80=94=20honest=20README=20claim=20+=20fail-loud=20gu?= =?UTF-8?q?ard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whole-branch review (Important, confirmed): README said 'no installs' but run-example shells the real gate whose probes need pytest; on stdlib-only python3 it died silently (set -e, exit 1, zero output) after clobbering the tracked verdict fixture. README now names the pytest requirement; the script checks for pytest up front and prints the uv one-liner before touching anything. --- examples/flaky-test-triage/README.md | 2 +- examples/flaky-test-triage/scripts/run-example | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/flaky-test-triage/README.md b/examples/flaky-test-triage/README.md index 2b05fa8..7a633f6 100644 --- a/examples/flaky-test-triage/README.md +++ b/examples/flaky-test-triage/README.md @@ -64,7 +64,7 @@ python3 -m loop metrics examples/flaky-test-triage ## Run it yourself One entrypoint re-derives the committed held-out verdict from a **live** gate run and checks the -terminal claim against it — no installs, ~1–2s: +terminal claim against it — needs `pytest` importable by `python3` (no other installs), ~1–2s: ```bash bash examples/flaky-test-triage/scripts/run-example diff --git a/examples/flaky-test-triage/scripts/run-example b/examples/flaky-test-triage/scripts/run-example index 83dabc1..529109a 100644 --- a/examples/flaky-test-triage/scripts/run-example +++ b/examples/flaky-test-triage/scripts/run-example @@ -5,6 +5,12 @@ set -euo pipefail EX="$(cd "$(dirname "$0")/.." && pwd)" REPO="$(cd "$EX/../.." && pwd)" +python3 -c "import pytest" 2>/dev/null || { + echo "run-example: the toy target's gate checks shell pytest — install it, or run:" >&2 + echo " uv run --with pytest bash examples/flaky-test-triage/scripts/run-example" >&2 + exit 1 +} + python3 -B "$REPO/scripts/holdout_gate.py" "$EX/target/manifest.json" --cwd "$EX/target" \ > "$EX/.loop/artifacts/holdout-verdict.json" From 9bc198d6fc0b96d69eb56bf5571c3cd4a6f11ca7 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Thu, 9 Jul 2026 20:31:52 -0400 Subject: [PATCH 10/10] =?UTF-8?q?chore(release):=200.8.0=20=E2=80=94=20com?= =?UTF-8?q?poses=20the=20field=20(ST3=20adapters=20+=20ST4=20funnel)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 15 ++++++++++++++- README.md | 10 +++++----- pyproject.toml | 2 +- scripts/test_docs_version.py | 5 +++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index fb996c9..d67e29a 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "loop-engineer", - "version": "0.7.0", + "version": "0.8.0", "description": "Design, launch, verify, repair, and improve agent loops. A Claude-Code-native architect+operator for long-running, verifiable, self-improving agentic-coding systems.", "author": { "name": "Sollan Systems", "url": "https://github.com/SollanSystems" }, "homepage": "https://github.com/SollanSystems/loop-engineer", diff --git a/CHANGELOG.md b/CHANGELOG.md index 896851b..546cbc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to `loop-engineer` are documented here. `WORKFLOW.md` and `README.md` are reworded to describe the mechanism; the 0.3.4 history is left intact. -## Unreleased +## 0.8.0 — 2026-07-09 **ST3 — integration adapters.** `loop/integrations.py`: an engine-neutral, pure-stdlib projection (`EngineOutcome` + `to_terminal_state`) from any @@ -32,6 +32,19 @@ recorded FCR-1.0 follow-up) and a Temporal recipe lands (visible-green/holdout-red → `FailedUnverifiable` with `false_completion: true`, never `Succeeded`) and pass the doctor round-trip. +**ST4 — contributor funnel.** `loop inspect` now recognizes a foreign +Superpowers-style run dir read-only (`loop/foreign.py` — a layout mapper onto +the existing `LoopPaths` seam; the M2/M3-hardened scorer is untouched and a +foreign harness with no gate and no terminal record scores honestly low). +The reading is checked in as `docs/gap-reports/superpowers.md` — the §14 +conformance checklist evaluated against a vendored, sanitized fixture +(`examples/superpowers-run/`). A second runnable example lands: +`examples/flaky-test-triage/` — doctor-clean, gate-backed, and the showcase +for repair records (`loop metrics` derives a non-null repair-productivity of +1.0 from its anchored red→green repair). Seven gate-backed starter issues are +drafted under `docs/contributing/issues/` and filed at release; +CONTRIBUTING gains the start-here funnel. + ## 0.7.0 — 2026-07-08 **ST2 — the portable standard.** The on-disk contract is now a documented, diff --git a/README.md b/README.md index 49b00ac..70a17f1 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![CI](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml) [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) -[![Release](https://img.shields.io/badge/release-0.7.0-blue)](https://github.com/SollanSystems/loop-engineer/tags) +[![Release](https://img.shields.io/badge/release-0.8.0-blue)](https://github.com/SollanSystems/loop-engineer/tags) Long-running agents commit **false completion**. After context compaction they forget what "done" meant, optimize to the visible test, patch in circles, and @@ -307,7 +307,7 @@ refuses an evidence-free `Succeeded` at write time. Recipe: **CI** — one workflow step validates the contract and publishes a scorecard: ```yaml -- uses: SollanSystems/loop-engineer@v0.7.0 +- uses: SollanSystems/loop-engineer@v0.8.0 with: path: "." ``` @@ -397,12 +397,12 @@ license, and README differentiation. ## Status -- Version: `0.7.0` -- Release tag: `v0.7.0` (PyPI publish trigger; plugin tags through 0.6.0 used `loop-engineer--v`) +- Version: `0.8.0` +- Release tag: `v0.8.0` (PyPI publish trigger; plugin tags through 0.6.0 used `loop-engineer--v`) - License: MIT - Primary interface: Claude Code plugin - Portable core: Python CLI + JSON schemas -- Current reference example: `examples/coverage-repair` +- Current reference examples: `examples/coverage-repair`, `examples/flaky-test-triage` --- diff --git a/pyproject.toml b/pyproject.toml index edac91d..7177e59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "loop-engineer" -version = "0.7.0" +version = "0.8.0" description = "Portable Loop Contract Core: validate and inspect repo-native operating contracts for agent loops." readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/test_docs_version.py b/scripts/test_docs_version.py index f09ea69..8a70eae 100644 --- a/scripts/test_docs_version.py +++ b/scripts/test_docs_version.py @@ -15,9 +15,9 @@ def test_readme_has_no_stale_seven_skills(): assert "all 9 skills" in readme -def test_plugin_version_is_0_7_0(): +def test_plugin_version_is_0_8_0(): plugin = json.loads(_read(".claude-plugin/plugin.json")) - assert plugin["version"] == "0.7.0" + assert plugin["version"] == "0.8.0" def test_pyproject_version_matches_plugin(): @@ -30,6 +30,7 @@ def test_pyproject_version_matches_plugin(): def test_changelog_has_current_and_historical_entries(): changelog = _read("CHANGELOG.md") + assert "## 0.8.0" in changelog assert "## 0.7.0" in changelog assert "## 0.6.1" in changelog assert "## 0.6.0" in changelog