diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cc6cea8..465aeb4 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "loop-engineer", - "version": "0.5.0", + "version": "0.6.0", "description": "Design, launch, verify, repair, and improve agent loops. A Claude-Code-native architect+operator for long-running, verifiable, self-improving agentic-coding systems.", "author": { "name": "Sollan Systems", "url": "https://github.com/SollanSystems" }, "homepage": "https://github.com/SollanSystems/loop-engineer", diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d81626..7e71faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,65 @@ All notable changes to `loop-engineer` are documented here. `WORKFLOW.md` and `README.md` are reworded to describe the mechanism; the 0.3.4 history is left intact. +## 0.6.0 — 2026-07-03 + +"Metrics real": false-completion-rate (FCR) and repair-productivity (RP) graduate +from claims to derivations (the ST1 spec), and the derivation itself survived two +rounds of adversarial red-teaming before merge — every exploit found is now a +pinned regression test. (PR #16.) + +### Added +- **`loop metrics `** — derives FCR and RP from a loop's real on-disk + evidence (RUNLOG, verify bundles, held-out verdict, repair records, receipts), + never from agent narration. FCR is computed two ways — the deterministic + claim×verify cross-join and the aggregated held-out `false_completion` flag — + and disagreement is surfaced, not resolved. An unmatched success claim counts + as a false completion (fail-closed). Output is a `loop-engineer/metrics@1` + scorecard whose `provenance` block names every input file, so a skeptic can + re-derive each number by hand. +- **`loop metrics --baseline`** — writes `docs/metrics-baseline.json` and + **refuses** (non-zero exit, writes nothing) unless the run is genuinely + gate-backed: a structurally valid held-out verdict artifact must exist (a gate + line in a verify script never qualifies); no rejected or unanchored repair + record; the two FCR methods must agree; a vacuous zero-claim run cannot + baseline. +- **Published baseline** over the gate-backed `examples/coverage-repair`: + **FCR 0.0, RP 1.0** — the README numbers cite the committed file (a test binds + the README literals to the JSON), reproducible with + `python3 -m loop metrics examples/coverage-repair`. +- **Canonical record schemas** — `schemas/repair-record.schema.json` + (`loop-engineer/repair@1`, RP's only input) and + `schemas/rollout-record.schema.json` (`loop-engineer/rollout@1`, the separate + candidate-adjudication artifact). Ends the two-shapes-both-called-"the repair + record" ambiguity; `validate_contract` checks record files when present and + `doctor` reports which record schemas it validated. +- **`loop` console script** (`[project.scripts]`) — the CLI runs from any + directory under the supported editable install. + +### Changed +- **`productive` is recomputed, never trusted.** `recheck_productive` recomputes + it from each record's own evidence and rejects disagreements; + `rollout_ledger.summarize()` (whose productivity key is now honestly named + `rollout_productivity`) and the metrics command aggregate only validated + records. Repair records additionally **anchor** to the deterministic verify + bundles: `verification_before/after` scores must match a same-task red→green + bundle pair (order-enforced when known), or the record is rejected/unanchored. +- **Claim semantics are outcome-class aware.** A completion-class claim + (`task_passed`/`succeeded`/`terminal`) is clean only if every verify bundle in + its iteration is green — no exceptions; a progress-class claim (`advanced`) + may carry a red intermediate only if the same task reaches green in a strictly + later iteration. Unrecognized outcome tokens are surfaced in provenance + instead of silently escaping the denominator. + +### Honesty hardening (adversarial pre-merge review) +Two red-team rounds (four, then two, adversarial reviewers) attacked the metrics +implementation before merge and confirmed 17 issues — including a `--baseline` +that would have published a clean headline FCR over a run its own held-out gate +had flagged, and an `evidence_backed` satisfiable by a prose mention of the +gate. All are fixed and pinned as regression tests; the honest residual is +documented in the README: a committed verdict artifact is *evidence, not proof* — +tamper detection belongs to the anti-cheat layer. + ## 0.5.0 — 2026-07-03 The two pre-launch milestones of the v1.0 roadmap landed together: **"enforce the diff --git a/README.md b/README.md index ee067c7..37cd16f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![CI](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/SollanSystems/loop-engineer/actions/workflows/ci.yml) [![Python 3.10–3.12](https://img.shields.io/badge/python-3.10%E2%80%933.12-blue)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) -[![Release](https://img.shields.io/badge/release-0.5.0-blue)](https://github.com/SollanSystems/loop-engineer/tags) +[![Release](https://img.shields.io/badge/release-0.6.0-blue)](https://github.com/SollanSystems/loop-engineer/tags) Long-running agents commit **false completion**. After context compaction they forget what "done" meant, optimize to the visible test, patch in circles, and @@ -362,8 +362,8 @@ license, and README differentiation. ## Status -- Version: `0.5.0` -- Release tag: `loop-engineer--v0.5.0` +- Version: `0.6.0` +- Release tag: `loop-engineer--v0.6.0` - License: MIT - Primary interface: Claude Code plugin - Portable core: Python CLI + JSON schemas diff --git a/pyproject.toml b/pyproject.toml index 85966bf..c81e90a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "loop-engineer" -version = "0.5.0" +version = "0.6.0" description = "Portable Loop Contract Core: validate and inspect repo-native operating contracts for agent loops." readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/test_docs_version.py b/scripts/test_docs_version.py index 4afae04..3af2a98 100644 --- a/scripts/test_docs_version.py +++ b/scripts/test_docs_version.py @@ -15,9 +15,9 @@ def test_readme_has_no_stale_seven_skills(): assert "all 9 skills" in readme -def test_plugin_version_is_0_5_0(): +def test_plugin_version_is_0_6_0(): plugin = json.loads(_read(".claude-plugin/plugin.json")) - assert plugin["version"] == "0.5.0" + assert plugin["version"] == "0.6.0" def test_pyproject_version_matches_plugin(): @@ -30,6 +30,7 @@ def test_pyproject_version_matches_plugin(): def test_changelog_has_current_and_historical_entries(): changelog = _read("CHANGELOG.md") + assert "## 0.6.0" in changelog assert "## 0.5.0" in changelog assert "## 0.3.4" in changelog assert "## 0.3.3" in changelog