From 7a0cbc9b763af94da35bc3b3150e0a865bfd86b8 Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Thu, 9 Jul 2026 22:52:22 -0400 Subject: [PATCH] =?UTF-8?q?fix(scaffold):=20metrics-clean=20RUNLOG=20seed?= =?UTF-8?q?=20=E2=80=94=20retire=20both=20recipes'=20inline=20resets=20(#4?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-07-08-v0.8.0-composes-the-field.md | 2 +- examples/langgraph-emit/graph_example.py | 6 --- examples/temporal-certify/workflow_example.py | 6 --- scripts/test_emit.py | 9 ++++ scripts/test_template_roundtrip.py | 17 +------ templates/RUNLOG.md.tmpl | 49 +++++++------------ 6 files changed, 31 insertions(+), 58 deletions(-) diff --git a/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md b/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md index 5cccd3f..44e79b8 100644 --- a/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md +++ b/docs/superpowers/plans/2026-07-08-v0.8.0-composes-the-field.md @@ -14,7 +14,7 @@ - **Two PRs:** PR-A = ST3 (Tasks 1–4), PR-B = ST4 + release cut (Tasks 5–10). PR-B branches from main after PR-A merges. The 0.8.0 release commit is the **last** commit of PR-B. - **Paths are the shipped reality, not the 2026-06-30 spec's:** helper = `loop/integrations.py` (NOT `loop_engineer.integrations`); docs = `docs/integrations/.md` (NOT `reference/integrations/`). -- **All disk writes go through `loop.emit`** (`open_contract` / `append_iteration` / `append_receipt` / `terminate`). No second write path for contract artifacts. (Recipes may write their own verify-bundle/verdict JSONs under `.loop/artifacts/` — those are evidence artifacts, not contract objects.) *Adjudicated carve-out (PR-A whole-branch review, 2026-07-08):* both recipes reset `/RUNLOG.md` to emit's own fresh header immediately after `open_contract`, because the scaffold seeds an unfilled `{{ITERATION_OUTCOME}}` placeholder that `loop metrics` flags as an unrecognized outcome token and the root fix (templates/scaffold/emit) is outside PR-A's file discipline. The reset strips a placeholder, never fabricates state. Root-fix affordance (e.g. `emit.open_contract(seed_runlog=False)`) is tracked as a PR-B contributor issue. +- **All disk writes go through `loop.emit`** (`open_contract` / `append_iteration` / `append_receipt` / `terminate`). No second write path for contract artifacts. (Recipes may write their own verify-bundle/verdict JSONs under `.loop/artifacts/` — those are evidence artifacts, not contract objects.) *Adjudicated carve-out (PR-A whole-branch review, 2026-07-08):* both recipes reset `/RUNLOG.md` to emit's own fresh header immediately after `open_contract`, because the scaffold seeds an unfilled `{{ITERATION_OUTCOME}}` placeholder that `loop metrics` flags as an unrecognized outcome token and the root fix (templates/scaffold/emit) is outside PR-A's file discipline. The reset strips a placeholder, never fabricates state. Root-fix affordance (e.g. `emit.open_contract(seed_runlog=False)`) is tracked as a PR-B contributor issue. **[Retired 2026-07-09 by the #40 root fix — `templates/RUNLOG.md.tmpl` seeds no `## Iteration` block, so a fresh scaffold is metrics-clean by construction; both recipe resets are removed and no `seed_runlog` kwarg was needed.]** - **PR-A file discipline:** `loop/` gains ONLY `integrations.py`. No modification to `scripts/holdout_gate.py`, `scripts/anticheat_scan.py`, `scripts/inspect_loop.py`, `scripts/metrics.py`, `scripts/self_eval.py`, `scripts/validate_frontmatter.py`, or anything under `schemas/`, `templates/`, `evals/`. `scripts/test_langgraph_recipe.py` stays **byte-identical** (new assertions go in a NEW file `scripts/test_langgraph_recipe_st3.py`). New test files under `scripts/` are additions, allowed. - **PR-B scorer discipline:** `scripts/inspect_loop.py` may change ONLY (a) the path-resolution seam (3 `resolve_loop_paths(...)` call sites → `_resolve_paths(...)`) and (b) additive report labeling (`foreign_layout`, `advisory`). Zero changes to `_CHECKS` weights, credit tiers, signal regexes, the score cap, or `_verdict`. A regression test pins that the fixture earns no invoked credit. - **Fixed precedence** in `to_terminal_state`: safety → human → blocked → budget → spec-gap → gate verdict. `FailedSafety` and `AbortedByHuman` beat a green gate. diff --git a/examples/langgraph-emit/graph_example.py b/examples/langgraph-emit/graph_example.py index a1625c1..e6152cb 100644 --- a/examples/langgraph-emit/graph_example.py +++ b/examples/langgraph-emit/graph_example.py @@ -99,12 +99,6 @@ def certify(state: State) -> dict: def main(workspace: str, sabotage: bool) -> int: emit.open_contract(workspace) - # The fresh scaffold's RUNLOG carries a placeholder example iteration (its - # unfilled outcome renders as "REPLACE"); start the demo's run history clean - # so `loop metrics` scores only the graph's own iterations. - (Path(workspace) / "RUNLOG.md").write_text( - f"# RUNLOG.md — {Path(workspace).name}\n", encoding="utf-8" - ) graph = ( StateGraph(State) .add_node(do_work) diff --git a/examples/temporal-certify/workflow_example.py b/examples/temporal-certify/workflow_example.py index 153d786..5db20e7 100644 --- a/examples/temporal-certify/workflow_example.py +++ b/examples/temporal-certify/workflow_example.py @@ -153,12 +153,6 @@ def certify_workflow_failure(workspace: str, cause: BaseException | None) -> dic async def run_and_certify(client: Client, workspace: str, *, sabotage: bool, wf_id: str) -> dict: emit.open_contract(workspace) - # The fresh scaffold's RUNLOG carries a placeholder example iteration (its - # unfilled outcome renders as "REPLACE"); start the demo's run history clean - # so `loop metrics` scores only the workflow's own iterations. - (Path(workspace) / "RUNLOG.md").write_text( - f"# RUNLOG.md — {Path(workspace).name}\n", encoding="utf-8" - ) try: return await client.execute_workflow( CertifiedGoalWorkflow.run, WorkArgs(workspace=workspace, sabotage=sabotage), diff --git a/scripts/test_emit.py b/scripts/test_emit.py index 4f54c10..4af3882 100644 --- a/scripts/test_emit.py +++ b/scripts/test_emit.py @@ -8,6 +8,7 @@ import pytest +import metrics from loop import emit from loop.contract import validate_contract @@ -24,6 +25,14 @@ def test_open_contract_is_doctor_clean(workspace): assert validate_contract(workspace)["ok"] is True +def test_open_contract_is_metrics_clean(workspace): + # The seeded RUNLOG is reference-only prose — no `## Iteration` block, so a + # fresh scaffold must score zero iterations and no outcome tokens. + scorecard = metrics.compute_metrics(workspace) + assert scorecard["provenance"]["unrecognized_outcomes"] == [] + assert scorecard["iterations_claiming_success"] == 0 + + def test_append_iteration_writes_parseable_runlog_and_updates_state(workspace): runlog = emit.append_iteration( workspace, iteration_id=1, outcome="task_passed", task_id="T1", diff --git a/scripts/test_template_roundtrip.py b/scripts/test_template_roundtrip.py index 41cb84d..a7dcbed 100644 --- a/scripts/test_template_roundtrip.py +++ b/scripts/test_template_roundtrip.py @@ -90,21 +90,8 @@ def _fill_values() -> dict[str, str]: "ARTIFACT_PATH": ".loop/artifacts/verify-T1.json", "FALSE_COMPLETION": "false", # boolean "TERMINAL_REASON": "fast gate passed with evidence", - # --- RUNLOG.md prose --- - "ITERATION_DATE": "2026-06-30", - "STATE_NAME": "running", - "ACTION_1": "ran the fast gate", - "ACTION_2": "recorded evidence", - "VERIFY_GATE": "verify-fast", - "VERIFY_OUTCOME": "PASS", - "VERIFY_CMD": "scripts/verify-fast", - "SCORE_BEFORE": "0.0", - "SCORE_AFTER": "1.0", - "ITERATION_OUTCOME": "task_passed", - "REPAIR_ACTION": "n/a", - "REPAIR_ATTEMPT": "0", - "IMPROVEMENT_FLAG": "n/a", - "ITERATION_NOTES": "clean first pass", + # RUNLOG.md seeds a reference-only preamble that declares only + # {{PROJECT_NAME}}, so it needs no per-iteration prose fill. # --- SPEC / WORKFLOW / AGENTS / EVALS prose --- "NON_GOAL_1": "no production writes", "NON_GOAL_2": "no schema changes", diff --git a/templates/RUNLOG.md.tmpl b/templates/RUNLOG.md.tmpl index 3c7d4fe..7eb3251 100644 --- a/templates/RUNLOG.md.tmpl +++ b/templates/RUNLOG.md.tmpl @@ -2,41 +2,30 @@ > Human-readable iteration history. Append-only — one entry per loop iteration. > Machine state lives in `.loop/state.json`; this is the audit trail. +> +> This file starts with no iterations. `loop metrics` reads only `## Iteration` +> blocks, so a fresh RUNLOG scores clean by construction. The entry format below +> is a reference, not an iteration — `loop.emit.append_iteration()` writes real +> blocks for you (or copy the shape by hand) beneath the marker at the bottom. --- -## Iteration {{ITERATION_ID}} — {{ITERATION_DATE}} +## Entry format (reference — not an iteration, not scored) -**State entered:** `{{STATE_NAME}}` -**Active task:** `{{ACTIVE_TASK_ID}}` — {{TASK_TITLE}} -**Plan version:** {{PLAN_VERSION}} +Each iteration is a section headed `## Iteration — ` carrying: -### Actions taken - -- {{ACTION_1}} -- {{ACTION_2}} - -### Verification result - -- **Gate:** `{{VERIFY_GATE}}` — {{VERIFY_OUTCOME}} (`{{VERIFY_CMD}}`) -- **Score before:** {{SCORE_BEFORE}} -- **Score after:** {{SCORE_AFTER}} - -### Outcome - -`{{ITERATION_OUTCOME}}` — one of: `task_passed`, `task_failed`, `repair_triggered`, `approval_requested`, `replanned`, `terminal` - -### Repair record (if repair triggered) - -- **Failure mode:** {{FAILURE_MODE}} -- **Repair action:** {{REPAIR_ACTION}} -- **Attempt:** {{REPAIR_ATTEMPT}} of {{REPAIR_CAP}} -- **Measurable improvement:** {{IMPROVEMENT_FLAG}} - -### Notes - -{{ITERATION_NOTES}} +- **State entered / Active task / Plan version** — the loop state, the task + worked, and the plan version in force. +- **Actions taken** — a bullet list of what the agent did. +- **Verification result** — the gate (`scripts/verify-*`), its outcome, and the + score before/after. +- **Outcome** — exactly one backticked token, one of: `task_passed`, + `task_failed`, `repair_triggered`, `approval_requested`, `replanned`, + `terminal`. +- **Repair record** — failure mode, repair action, attempt N of the cap, and the + measurable-improvement flag (only when a repair was triggered). +- **Notes** — free-form context. --- - +