From 5fa44492c5f1084c9b5b5f453001aae451c8c470 Mon Sep 17 00:00:00 2001 From: Paul Jialiang Wu Date: Fri, 21 Aug 2026 11:51:01 -0500 Subject: [PATCH] fix(rubric): make the 23/23 run reproducible instead of merely asserted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An independent review of the article about this work found the disqualifying defect, and it was mine: the first rubric — the one that scored 23/23 — was never committed. `git log --all -- docs/rubrics/` returned exactly one commit. The only trace of 23/23 anywhere was PROSE written by the person claiming it: a commit message, a YAML comment, a report card. That is precisely the category this rubric exists to reject. Evidence is observed, never asserted — and the headline number of the whole exercise was asserted. So it is now observable: docs/rubrics/archive/agent-loop-engineering.v1.yml 26 items (23 verifiable + 3 gaps) docs/rubrics/archive/AUDIT.v1.md its output: 23/23, gate PASS python3 scripts/audit_loop_rubric.py \ --rubric docs/rubrics/archive/agent-loop-engineering.v1.yml The file is labelled a RECONSTRUCTION, not a recovered original: it is the current rubric minus the five second-pass items, committed after the fact. Saying so is the difference between evidence and a better-dressed assertion. `--rubric` is added to the checker for this, and the report now names which rubric it scored, so an archived run cannot be mistaken for the live one. Also fixes a dead pointer the same review caught: both files cited `docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md`, a path that does not exist in this repo (the playbook ships with the anyagent skill). The rules are now restated in place so each file stands alone. 606 tests pass, ruff clean. Co-Authored-By: Claude Opus 5 (1M context) --- docs/rubrics/AUDIT.md | 6 +- docs/rubrics/agent-loop-engineering.yml | 3 +- docs/rubrics/archive/AUDIT.v1.md | 94 ++++++++++ .../archive/agent-loop-engineering.v1.yml | 174 ++++++++++++++++++ scripts/audit_loop_rubric.py | 15 +- 5 files changed, 283 insertions(+), 9 deletions(-) create mode 100644 docs/rubrics/archive/AUDIT.v1.md create mode 100644 docs/rubrics/archive/agent-loop-engineering.v1.yml diff --git a/docs/rubrics/AUDIT.md b/docs/rubrics/AUDIT.md index 27979f1..0372e8a 100644 --- a/docs/rubrics/AUDIT.md +++ b/docs/rubrics/AUDIT.md @@ -31,7 +31,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `src/loopeng/adapters/base.py:62` - **L8-maker-not-checker** · 10 — evaluation must be trustworthy - claim: The thing that builds is not the thing that grades. - - evidence: `tests/test_maker_checker.py -> 30 passed in 0.16s` + - evidence: `tests/test_maker_checker.py -> 30 passed in 0.13s` - **L9-reflection** · 4/6 — Plan-Execute-Observe-Reflect, Self-Reflection - claim: Why the last attempt scored what it did is carried into the next attempt. - evidence: `src/loopeng/adapters/base.py:66` @@ -43,7 +43,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `src/loopeng/adapters/base.py:142` - **L12-recovery-state** · 8 — Recovery keeps enough state to resume - claim: A failed change can be rolled back rather than restarting from zero. - - evidence: `tests/test_checkpoint.py -> 2 passed in 0.50s` + - evidence: `tests/test_checkpoint.py -> 2 passed in 0.61s` - **L13-exit-success** · 9 — explicit exits: success - claim: The loop stops when the goal is verifiably met. - evidence: `src/loopeng/loop/convergence.py:29` @@ -79,7 +79,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `tests/test_proof.py -> 7 passed in 0.09s` - **L29-success-rate-metric** · 14 — measure tokens, time, iteration count AND task success rate - claim: Task success RATE is computed across runs, not just per-run outcome. - - evidence: `tests/test_success_rate.py -> 6 passed in 0.09s` + - evidence: `tests/test_success_rate.py -> 6 passed in 0.07s` - **L31-error-memory** · 13 — record errors so the agent does not repeat the same mistake - claim: Failures that recur across runs are surfaced to the next attempt. - evidence: `src/loopeng/memory/store.py:277` diff --git a/docs/rubrics/agent-loop-engineering.yml b/docs/rubrics/agent-loop-engineering.yml index a19bdb0..2e4ca3b 100644 --- a/docs/rubrics/agent-loop-engineering.yml +++ b/docs/rubrics/agent-loop-engineering.yml @@ -7,7 +7,8 @@ # THIS FILE IS THE SPEC. `scripts/audit_loop_rubric.py` is the only consumer, so the # published score cannot drift from the claims it scores. # -# RULES (docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md): +# RULES (the operationalizing-a-rubric playbook; it ships with the anyagent skill, not in +# this repo -- the rules are restated here so the file stands alone): # * Evidence is OBSERVED (a symbol in shipped code, a passing test), never claimed. # * No evidence => NO. Not "probably". # * A probe that cannot run is `unmeasured` — excluded from the rate, and it BLOCKS diff --git a/docs/rubrics/archive/AUDIT.v1.md b/docs/rubrics/archive/AUDIT.v1.md new file mode 100644 index 0000000..1a5b979 --- /dev/null +++ b/docs/rubrics/archive/AUDIT.v1.md @@ -0,0 +1,94 @@ +# Agent Loop Engineering — conformance audit + +Rubric: `docs/rubrics/archive/agent-loop-engineering.v1.yml` — 26 items drawn from *Agent Loop Engineering — 讲座总结* (DataApplab / AI聘 (info@aipin.io), received 2026-08-20). + +That source is a **lecture summary; taxonomy, no verification against a running system**. Every claim below is scored against shipped code or a test that was actually executed. No evidence means no. + +**Conformance: 23/23 (100%)** verifiable claims implemented · 3 declared gaps · 0 unmeasured · gate **PASS** + +## Implemented — with observed evidence + +- **L1-loop-not-oneshot** · 2 — why an agent needs a loop + - claim: The system iterates plan→act→observe→evaluate rather than answering once. + - evidence: `src/loopeng/loop/controller.py:75` +- **L2-goal** · 3 — Goal + - claim: The loop carries an explicit goal and a definition of done. + - evidence: `src/loopeng/config.py:89` +- **L3-state** · 3 — State + - claim: Progress lives outside the model: steps taken, results, environment. + - evidence: `src/loopeng/memory/store.py:196` +- **L4-policy** · 3 — Policy + - claim: Something decides the next action from the current state. + - evidence: `src/loopeng/adapters/base.py:103` +- **L5-action-space** · 3 — Action space + - claim: The agent's available actions are declared, not open-ended. + - evidence: `src/loopeng/adapters/base.py:52` +- **L6-observation** · 3 — Observation + - claim: Every action returns structured environment feedback. + - evidence: `src/loopeng/adapters/base.py:16` +- **L7-evaluation** · 3/10 — Evaluation is the controller + - claim: A verifiable evaluator decides progress, not the maker's self-report. + - evidence: `src/loopeng/adapters/base.py:62` +- **L8-maker-not-checker** · 10 — evaluation must be trustworthy + - claim: The thing that builds is not the thing that grades. + - evidence: `tests/test_maker_checker.py -> 30 passed in 0.14s` +- **L9-reflection** · 4/6 — Plan-Execute-Observe-Reflect, Self-Reflection + - claim: Why the last attempt scored what it did is carried into the next attempt. + - evidence: `src/loopeng/adapters/base.py:66` +- **L10-replan-on-plateau** · 4 — the plan is not immutable + - claim: Feedback can force a change of strategy, not just another attempt. + - evidence: `src/loopeng/config.py:100` +- **L11-retry-transient-only** · 8 — Retry vs Recovery + - claim: Only retryable (infrastructure) failures are retried; not every error. + - evidence: `src/loopeng/adapters/base.py:142` +- **L12-recovery-state** · 8 — Recovery keeps enough state to resume + - claim: A failed change can be rolled back rather than restarting from zero. + - evidence: `tests/test_checkpoint.py -> 2 passed in 0.49s` +- **L13-exit-success** · 9 — explicit exits: success + - claim: The loop stops when the goal is verifiably met. + - evidence: `src/loopeng/loop/convergence.py:29` +- **L14-exit-budget** · 9/14 — explicit exits: budget (iterations, tokens, wall clock) + - claim: The loop stops on a spent budget, and the budget has more than one dimension. + - evidence: `src/loopeng/loop/convergence.py:38` +- **L15-exit-giveup** · 9 — explicit exits: give up and report + - claim: Repeated non-progress ends the run and reports failure instead of looping. + - evidence: `src/loopeng/loop/convergence.py:36` +- **L16-safety-terminal** · 13 — permission control + - claim: A safety failure is terminal and unbypassable, whatever the score. + - evidence: `src/loopeng/loop/convergence.py:30` +- **L17-permission-boundary** · 13 — which tools/data the agent may touch is limited + - claim: Execution is jailed and shell metacharacters are refused. + - evidence: `src/loopeng/adapters/safety.py:5` +- **L18-human-in-the-loop** · 12 — Human-in-the-Loop for high-risk actions + - claim: High-risk completion requires a human, and the caller cannot self-approve. + - evidence: `src/loopeng/config.py:127` +- **L19-hitl-unbypassable** · 12 — the gate must actually hold + - claim: An unattended run cannot pre-confirm its own result. + - evidence: `tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate -> 4 passed in 0.05s` +- **L20-multi-agent-graph** · 11 — Loop becomes Graph with many agents + - claim: Multiple agents are coordinated as a dependency graph, cycles refused. + - evidence: `src/loopeng/orchestration/coordinator.py:6` +- **L21-observability** · 13 — every iteration leaves enough log/trace to debug + - claim: A run is reconstructable after the fact from recorded evidence. + - evidence: `src/loopeng/autonomous/report.py:37` +- **L22-token-accounting** · 14 — Token economics: measure tokens, time, iterations, success + - claim: Token cost, wall time and iteration count are recorded per run. + - evidence: `src/loopeng/proof.py:108` +- **L23-cost-never-faked** · 14 — measurement must be real to be useful + - claim: An unavailable cost is omitted, never estimated into the record. + - evidence: `tests/test_proof.py -> 7 passed in 0.09s` + +## Declared gaps — the engine does NOT do these, and says so + +Each probe passes while the gap is real and fails the moment it silently closes, so this list cannot quietly go stale. + +- **L24-typed-failure-taxonomy** · 8 — classify the error before choosing a response + - claim: Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout). + - status: **confirmed absent** — Only infra-vs-clean is distinguished. External eval item 3. +- **L25-branching-search** · 7 — Tree of Thoughts + - claim: The loop explores multiple candidate paths and prunes them. + - status: **confirmed absent** — Single-path refine with a dimension pivot. Deliberate: branching multiplies cost. +- **L26-trace-schema** · 13 — observability as a first-class trace + - claim: Runs emit a structured trace (spans/trace ids), not just a report. + - status: **confirmed absent** — Reports and proof packs exist; a span-level trace schema does not. + diff --git a/docs/rubrics/archive/agent-loop-engineering.v1.yml b/docs/rubrics/archive/agent-loop-engineering.v1.yml new file mode 100644 index 0000000..f18a91e --- /dev/null +++ b/docs/rubrics/archive/agent-loop-engineering.v1.yml @@ -0,0 +1,174 @@ +# ARCHIVED RECONSTRUCTION — the 23/23 rubric, committed after the fact. +# +# This is the FIRST version of docs/rubrics/agent-loop-engineering.yml: 26 items, +# 23 verifiable + 3 declared gaps. Run against the engine it scored 23/23, gate green, +# and that perfect score was the bug — every question here was written by someone who +# had just spent an hour in the codebase, and not one was expected to come back red. +# +# HONEST PROVENANCE: this file was never committed at the time. It is reconstructed by +# removing the five items added in the second pass, and committed now so the 23/23 claim +# is REPRODUCIBLE rather than merely asserted by its author — which is the whole point of +# the exercise it documents. Reconstruction, not a recovered original. +# +# Reproduce: python3 scripts/audit_loop_rubric.py --rubric docs/rubrics/archive/agent-loop-engineering.v1.yml +# +# Agent Loop Engineering — the lecture's claims, as a machine-checkable rubric. +# +# SOURCE: "AI开发进入新时代：Agent Loop Engineering - 讲座总结" (DataApplab / AI聘 webinar +# summary, emailed 2026-08-20). Sixteen sections describing what a mature agent loop +# must have. The essay asserts; nothing in it is verified against a running system. +# +# THIS FILE IS THE SPEC. `scripts/audit_loop_rubric.py` is the only consumer, so the +# published score cannot drift from the claims it scores. +# +# RULES (docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md): +# * Evidence is OBSERVED (a symbol in shipped code, a passing test), never claimed. +# * No evidence => NO. Not "probably". +# * A probe that cannot run is `unmeasured` — excluded from the rate, and it BLOCKS +# a green gate. An unmeasured item is never a silent pass. +# * `expect: absent` items are honest gap declarations: the rubric asserts the engine +# does NOT have this, and the probe FAILS if the gap ever silently closes. +version: 1 +source: + title: "Agent Loop Engineering — 讲座总结" + publisher: "DataApplab / AI聘 (info@aipin.io)" + received: "2026-08-20" + nature: "lecture summary; taxonomy, no verification against a running system" + +items: + - id: L1-loop-not-oneshot + section: "2 — why an agent needs a loop" + claim: "The system iterates plan→act→observe→evaluate rather than answering once." + probe: {type: grep, pattern: "class LoopController", paths: ["src/loopeng/loop/controller.py"]} + + - id: L2-goal + section: "3 — Goal" + claim: "The loop carries an explicit goal and a definition of done." + probe: {type: grep, pattern: "target_grade|target_score", paths: ["src/loopeng/config.py"]} + + - id: L3-state + section: "3 — State" + claim: "Progress lives outside the model: steps taken, results, environment." + probe: {type: grep, pattern: "def record_iteration", paths: ["src/loopeng/memory/store.py"]} + + - id: L4-policy + section: "3 — Policy" + claim: "Something decides the next action from the current state." + probe: {type: grep, pattern: "class RefactorBrief|def build_brief", paths: ["src/loopeng/adapters/base.py", "src/loopeng/loop/refactor_brief.py"]} + + - id: L5-action-space + section: "3 — Action space" + claim: "The agent's available actions are declared, not open-ended." + probe: {type: grep, pattern: "class Factory|class Refiner|class Judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L6-observation + section: "3 — Observation" + claim: "Every action returns structured environment feedback." + probe: {type: grep, pattern: "class Verdict", paths: ["src/loopeng/adapters/base.py"]} + + - id: L7-evaluation + section: "3/10 — Evaluation is the controller" + claim: "A verifiable evaluator decides progress, not the maker's self-report." + probe: {type: grep, pattern: "def judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L8-maker-not-checker + section: "10 — evaluation must be trustworthy" + claim: "The thing that builds is not the thing that grades." + probe: {type: pytest, node: "tests/test_maker_checker.py"} + + - id: L9-reflection + section: "4/6 — Plan-Execute-Observe-Reflect, Self-Reflection" + claim: "Why the last attempt scored what it did is carried into the next attempt." + probe: {type: grep, pattern: "class ReflectionContext", paths: ["src/loopeng/adapters/base.py"]} + + - id: L10-replan-on-plateau + section: "4 — the plan is not immutable" + claim: "Feedback can force a change of strategy, not just another attempt." + probe: {type: grep, pattern: "plateau_pivots", paths: ["src/loopeng/config.py"]} + + - id: L11-retry-transient-only + section: "8 — Retry vs Recovery" + claim: "Only retryable (infrastructure) failures are retried; not every error." + probe: {type: grep, pattern: "last_infra_failure", paths: ["src/loopeng/adapters/base.py"]} + + - id: L12-recovery-state + section: "8 — Recovery keeps enough state to resume" + claim: "A failed change can be rolled back rather than restarting from zero." + probe: {type: pytest, node: "tests/test_checkpoint.py"} + + - id: L13-exit-success + section: "9 — explicit exits: success" + claim: "The loop stops when the goal is verifiably met." + probe: {type: grep, pattern: "CONVERGED", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L14-exit-budget + section: "9/14 — explicit exits: budget (iterations, tokens, wall clock)" + claim: "The loop stops on a spent budget, and the budget has more than one dimension." + probe: {type: grep, pattern: "ITERATION_CAP|TOKEN_CAP|WALL_CAP", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L15-exit-giveup + section: "9 — explicit exits: give up and report" + claim: "Repeated non-progress ends the run and reports failure instead of looping." + probe: {type: grep, pattern: "PLATEAU", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L16-safety-terminal + section: "13 — permission control" + claim: "A safety failure is terminal and unbypassable, whatever the score." + probe: {type: grep, pattern: "BLOCKED_SAFETY", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L17-permission-boundary + section: "13 — which tools/data the agent may touch is limited" + claim: "Execution is jailed and shell metacharacters are refused." + probe: {type: grep, pattern: "within_workspace|shell=False", paths: ["src/loopeng/adapters/safety.py"]} + + - id: L18-human-in-the-loop + section: "12 — Human-in-the-Loop for high-risk actions" + claim: "High-risk completion requires a human, and the caller cannot self-approve." + probe: {type: grep, pattern: "class VerificationGate", paths: ["src/loopeng/config.py"]} + + - id: L19-hitl-unbypassable + section: "12 — the gate must actually hold" + claim: "An unattended run cannot pre-confirm its own result." + probe: {type: pytest, node: "tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate"} + + - id: L20-multi-agent-graph + section: "11 — Loop becomes Graph with many agents" + claim: "Multiple agents are coordinated as a dependency graph, cycles refused." + probe: {type: grep, pattern: "Kahn|cycle", paths: ["src/loopeng/orchestration/coordinator.py"]} + + - id: L21-observability + section: "13 — every iteration leaves enough log/trace to debug" + claim: "A run is reconstructable after the fact from recorded evidence." + probe: {type: grep, pattern: "def render_report", paths: ["src/loopeng/autonomous/report.py"]} + + - id: L22-token-accounting + section: "14 — Token economics: measure tokens, time, iterations, success" + claim: "Token cost, wall time and iteration count are recorded per run." + probe: {type: grep, pattern: "token_cost", paths: ["src/loopeng/proof.py"]} + + - id: L23-cost-never-faked + section: "14 — measurement must be real to be useful" + claim: "An unavailable cost is omitted, never estimated into the record." + probe: {type: pytest, node: "tests/test_proof.py"} + + # ---- honest gap declarations: the rubric asserts these are ABSENT ---- + - id: L24-typed-failure-taxonomy + section: "8 — classify the error before choosing a response" + claim: "Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout)." + expect: absent + probe: {type: grep, pattern: "class FailureType|REFEREE_UNAVAILABLE", paths: ["src/loopeng"]} + gap_note: "Only infra-vs-clean is distinguished. External eval item 3." + + - id: L25-branching-search + section: "7 — Tree of Thoughts" + claim: "The loop explores multiple candidate paths and prunes them." + expect: absent + probe: {type: grep, pattern: "tree_of_thought|beam_search", paths: ["src/loopeng"]} + gap_note: "Single-path refine with a dimension pivot. Deliberate: branching multiplies cost." + + - id: L26-trace-schema + section: "13 — observability as a first-class trace" + claim: "Runs emit a structured trace (spans/trace ids), not just a report." + expect: absent + probe: {type: grep, pattern: "opentelemetry|trace_id|def emit_span", paths: ["src/loopeng"]} + gap_note: "Reports and proof packs exist; a span-level trace schema does not." diff --git a/scripts/audit_loop_rubric.py b/scripts/audit_loop_rubric.py index 50721ef..82acf91 100644 --- a/scripts/audit_loop_rubric.py +++ b/scripts/audit_loop_rubric.py @@ -6,7 +6,8 @@ running system. This script is the missing half: every claim becomes a probe against shipped code or a real test run, and the score is whatever the probes return. -Discipline (playbook: operationalizing-a-paper-rubric-checklist-standard): +Discipline (the operationalizing-a-rubric playbook, which ships with the anyagent +skill rather than in this repo -- summarised here so this file stands alone): * Evidence is OBSERVED -- a symbol at a file:line, or a pytest node that actually passes. Never a claim in prose. @@ -98,8 +99,9 @@ def run_item(item: dict) -> dict: return {**item, "status": PASS if ok else FAIL, "evidence": evidence} -def audit() -> dict: - doc = yaml.safe_load(RUBRIC.read_text(encoding="utf-8")) +def audit(rubric: pathlib.Path | None = None) -> dict: + path = rubric or RUBRIC + doc = yaml.safe_load(path.read_text(encoding="utf-8")) results = [run_item(i) for i in doc["items"]] verifiable = [r for r in results if not r.get("expect")] @@ -113,6 +115,7 @@ def audit() -> dict: gate_ok = not missing and not unmeasured and all(g["status"] == PASS for g in gaps) return { + "rubric": str(path.relative_to(ROOT)), "source": doc["source"], "total_items": len(results), "verifiable": len(verifiable), @@ -129,7 +132,7 @@ def audit() -> dict: def render(a: dict) -> str: src = a["source"] L = ["# Agent Loop Engineering — conformance audit", "", - f"Rubric: `docs/rubrics/agent-loop-engineering.yml` — {a['total_items']} items drawn from " + f"Rubric: `{a['rubric']}` — {a['total_items']} items drawn from " f"*{src['title']}* ({src['publisher']}, received {src['received']}).", "", f"That source is a **{src['nature']}**. Every claim below is scored against shipped " "code or a test that was actually executed. No evidence means no.", "", @@ -169,12 +172,14 @@ def render(a: dict) -> str: def main() -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--rubric", default=None, + help="score an alternative rubric file (e.g. an archived version)") ap.add_argument("--json", action="store_true") ap.add_argument("--gate", action="store_true", help="exit 1 unless every item passes") ap.add_argument("--out") args = ap.parse_args() - a = audit() + a = audit(pathlib.Path(args.rubric).resolve() if args.rubric else None) text = json.dumps(a, indent=2) if args.json else render(a) if args.out: pathlib.Path(args.out).write_text(text + "\n", encoding="utf-8")