diff --git a/docs/rubrics/AUDIT.md b/docs/rubrics/AUDIT.md index 27979f1..0372e8a 100644 --- a/docs/rubrics/AUDIT.md +++ b/docs/rubrics/AUDIT.md @@ -31,7 +31,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `src/loopeng/adapters/base.py:62` - **L8-maker-not-checker** · 10 — evaluation must be trustworthy - claim: The thing that builds is not the thing that grades. - - evidence: `tests/test_maker_checker.py -> 30 passed in 0.16s` + - evidence: `tests/test_maker_checker.py -> 30 passed in 0.13s` - **L9-reflection** · 4/6 — Plan-Execute-Observe-Reflect, Self-Reflection - claim: Why the last attempt scored what it did is carried into the next attempt. - evidence: `src/loopeng/adapters/base.py:66` @@ -43,7 +43,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `src/loopeng/adapters/base.py:142` - **L12-recovery-state** · 8 — Recovery keeps enough state to resume - claim: A failed change can be rolled back rather than restarting from zero. - - evidence: `tests/test_checkpoint.py -> 2 passed in 0.50s` + - evidence: `tests/test_checkpoint.py -> 2 passed in 0.61s` - **L13-exit-success** · 9 — explicit exits: success - claim: The loop stops when the goal is verifiably met. - evidence: `src/loopeng/loop/convergence.py:29` @@ -79,7 +79,7 @@ That source is a **lecture summary; taxonomy, no verification against a running - evidence: `tests/test_proof.py -> 7 passed in 0.09s` - **L29-success-rate-metric** · 14 — measure tokens, time, iteration count AND task success rate - claim: Task success RATE is computed across runs, not just per-run outcome. - - evidence: `tests/test_success_rate.py -> 6 passed in 0.09s` + - evidence: `tests/test_success_rate.py -> 6 passed in 0.07s` - **L31-error-memory** · 13 — record errors so the agent does not repeat the same mistake - claim: Failures that recur across runs are surfaced to the next attempt. - evidence: `src/loopeng/memory/store.py:277` diff --git a/docs/rubrics/agent-loop-engineering.yml b/docs/rubrics/agent-loop-engineering.yml index a19bdb0..2e4ca3b 100644 --- a/docs/rubrics/agent-loop-engineering.yml +++ b/docs/rubrics/agent-loop-engineering.yml @@ -7,7 +7,8 @@ # THIS FILE IS THE SPEC. `scripts/audit_loop_rubric.py` is the only consumer, so the # published score cannot drift from the claims it scores. # -# RULES (docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md): +# RULES (the operationalizing-a-rubric playbook; it ships with the anyagent skill, not in +# this repo -- the rules are restated here so the file stands alone): # * Evidence is OBSERVED (a symbol in shipped code, a passing test), never claimed. # * No evidence => NO. Not "probably". # * A probe that cannot run is `unmeasured` — excluded from the rate, and it BLOCKS diff --git a/docs/rubrics/archive/AUDIT.v1.md b/docs/rubrics/archive/AUDIT.v1.md new file mode 100644 index 0000000..1a5b979 --- /dev/null +++ b/docs/rubrics/archive/AUDIT.v1.md @@ -0,0 +1,94 @@ +# Agent Loop Engineering — conformance audit + +Rubric: `docs/rubrics/archive/agent-loop-engineering.v1.yml` — 26 items drawn from *Agent Loop Engineering — 讲座总结* (DataApplab / AI聘 (info@aipin.io), received 2026-08-20). + +That source is a **lecture summary; taxonomy, no verification against a running system**. Every claim below is scored against shipped code or a test that was actually executed. No evidence means no. + +**Conformance: 23/23 (100%)** verifiable claims implemented · 3 declared gaps · 0 unmeasured · gate **PASS** + +## Implemented — with observed evidence + +- **L1-loop-not-oneshot** · 2 — why an agent needs a loop + - claim: The system iterates plan→act→observe→evaluate rather than answering once. + - evidence: `src/loopeng/loop/controller.py:75` +- **L2-goal** · 3 — Goal + - claim: The loop carries an explicit goal and a definition of done. + - evidence: `src/loopeng/config.py:89` +- **L3-state** · 3 — State + - claim: Progress lives outside the model: steps taken, results, environment. + - evidence: `src/loopeng/memory/store.py:196` +- **L4-policy** · 3 — Policy + - claim: Something decides the next action from the current state. + - evidence: `src/loopeng/adapters/base.py:103` +- **L5-action-space** · 3 — Action space + - claim: The agent's available actions are declared, not open-ended. + - evidence: `src/loopeng/adapters/base.py:52` +- **L6-observation** · 3 — Observation + - claim: Every action returns structured environment feedback. + - evidence: `src/loopeng/adapters/base.py:16` +- **L7-evaluation** · 3/10 — Evaluation is the controller + - claim: A verifiable evaluator decides progress, not the maker's self-report. + - evidence: `src/loopeng/adapters/base.py:62` +- **L8-maker-not-checker** · 10 — evaluation must be trustworthy + - claim: The thing that builds is not the thing that grades. + - evidence: `tests/test_maker_checker.py -> 30 passed in 0.14s` +- **L9-reflection** · 4/6 — Plan-Execute-Observe-Reflect, Self-Reflection + - claim: Why the last attempt scored what it did is carried into the next attempt. + - evidence: `src/loopeng/adapters/base.py:66` +- **L10-replan-on-plateau** · 4 — the plan is not immutable + - claim: Feedback can force a change of strategy, not just another attempt. + - evidence: `src/loopeng/config.py:100` +- **L11-retry-transient-only** · 8 — Retry vs Recovery + - claim: Only retryable (infrastructure) failures are retried; not every error. + - evidence: `src/loopeng/adapters/base.py:142` +- **L12-recovery-state** · 8 — Recovery keeps enough state to resume + - claim: A failed change can be rolled back rather than restarting from zero. + - evidence: `tests/test_checkpoint.py -> 2 passed in 0.49s` +- **L13-exit-success** · 9 — explicit exits: success + - claim: The loop stops when the goal is verifiably met. + - evidence: `src/loopeng/loop/convergence.py:29` +- **L14-exit-budget** · 9/14 — explicit exits: budget (iterations, tokens, wall clock) + - claim: The loop stops on a spent budget, and the budget has more than one dimension. + - evidence: `src/loopeng/loop/convergence.py:38` +- **L15-exit-giveup** · 9 — explicit exits: give up and report + - claim: Repeated non-progress ends the run and reports failure instead of looping. + - evidence: `src/loopeng/loop/convergence.py:36` +- **L16-safety-terminal** · 13 — permission control + - claim: A safety failure is terminal and unbypassable, whatever the score. + - evidence: `src/loopeng/loop/convergence.py:30` +- **L17-permission-boundary** · 13 — which tools/data the agent may touch is limited + - claim: Execution is jailed and shell metacharacters are refused. + - evidence: `src/loopeng/adapters/safety.py:5` +- **L18-human-in-the-loop** · 12 — Human-in-the-Loop for high-risk actions + - claim: High-risk completion requires a human, and the caller cannot self-approve. + - evidence: `src/loopeng/config.py:127` +- **L19-hitl-unbypassable** · 12 — the gate must actually hold + - claim: An unattended run cannot pre-confirm its own result. + - evidence: `tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate -> 4 passed in 0.05s` +- **L20-multi-agent-graph** · 11 — Loop becomes Graph with many agents + - claim: Multiple agents are coordinated as a dependency graph, cycles refused. + - evidence: `src/loopeng/orchestration/coordinator.py:6` +- **L21-observability** · 13 — every iteration leaves enough log/trace to debug + - claim: A run is reconstructable after the fact from recorded evidence. + - evidence: `src/loopeng/autonomous/report.py:37` +- **L22-token-accounting** · 14 — Token economics: measure tokens, time, iterations, success + - claim: Token cost, wall time and iteration count are recorded per run. + - evidence: `src/loopeng/proof.py:108` +- **L23-cost-never-faked** · 14 — measurement must be real to be useful + - claim: An unavailable cost is omitted, never estimated into the record. + - evidence: `tests/test_proof.py -> 7 passed in 0.09s` + +## Declared gaps — the engine does NOT do these, and says so + +Each probe passes while the gap is real and fails the moment it silently closes, so this list cannot quietly go stale. + +- **L24-typed-failure-taxonomy** · 8 — classify the error before choosing a response + - claim: Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout). + - status: **confirmed absent** — Only infra-vs-clean is distinguished. External eval item 3. +- **L25-branching-search** · 7 — Tree of Thoughts + - claim: The loop explores multiple candidate paths and prunes them. + - status: **confirmed absent** — Single-path refine with a dimension pivot. Deliberate: branching multiplies cost. +- **L26-trace-schema** · 13 — observability as a first-class trace + - claim: Runs emit a structured trace (spans/trace ids), not just a report. + - status: **confirmed absent** — Reports and proof packs exist; a span-level trace schema does not. + diff --git a/docs/rubrics/archive/agent-loop-engineering.v1.yml b/docs/rubrics/archive/agent-loop-engineering.v1.yml new file mode 100644 index 0000000..f18a91e --- /dev/null +++ b/docs/rubrics/archive/agent-loop-engineering.v1.yml @@ -0,0 +1,174 @@ +# ARCHIVED RECONSTRUCTION — the 23/23 rubric, committed after the fact. +# +# This is the FIRST version of docs/rubrics/agent-loop-engineering.yml: 26 items, +# 23 verifiable + 3 declared gaps. Run against the engine it scored 23/23, gate green, +# and that perfect score was the bug — every question here was written by someone who +# had just spent an hour in the codebase, and not one was expected to come back red. +# +# HONEST PROVENANCE: this file was never committed at the time. It is reconstructed by +# removing the five items added in the second pass, and committed now so the 23/23 claim +# is REPRODUCIBLE rather than merely asserted by its author — which is the whole point of +# the exercise it documents. Reconstruction, not a recovered original. +# +# Reproduce: python3 scripts/audit_loop_rubric.py --rubric docs/rubrics/archive/agent-loop-engineering.v1.yml +# +# Agent Loop Engineering — the lecture's claims, as a machine-checkable rubric. +# +# SOURCE: "AI开发进入新时代：Agent Loop Engineering - 讲座总结" (DataApplab / AI聘 webinar +# summary, emailed 2026-08-20). Sixteen sections describing what a mature agent loop +# must have. The essay asserts; nothing in it is verified against a running system. +# +# THIS FILE IS THE SPEC. `scripts/audit_loop_rubric.py` is the only consumer, so the +# published score cannot drift from the claims it scores. +# +# RULES (docs/skill/playbooks/operationalizing-a-paper-rubric-checklist-standa.md): +# * Evidence is OBSERVED (a symbol in shipped code, a passing test), never claimed. +# * No evidence => NO. Not "probably". +# * A probe that cannot run is `unmeasured` — excluded from the rate, and it BLOCKS +# a green gate. An unmeasured item is never a silent pass. +# * `expect: absent` items are honest gap declarations: the rubric asserts the engine +# does NOT have this, and the probe FAILS if the gap ever silently closes. +version: 1 +source: + title: "Agent Loop Engineering — 讲座总结" + publisher: "DataApplab / AI聘 (info@aipin.io)" + received: "2026-08-20" + nature: "lecture summary; taxonomy, no verification against a running system" + +items: + - id: L1-loop-not-oneshot + section: "2 — why an agent needs a loop" + claim: "The system iterates plan→act→observe→evaluate rather than answering once." + probe: {type: grep, pattern: "class LoopController", paths: ["src/loopeng/loop/controller.py"]} + + - id: L2-goal + section: "3 — Goal" + claim: "The loop carries an explicit goal and a definition of done." + probe: {type: grep, pattern: "target_grade|target_score", paths: ["src/loopeng/config.py"]} + + - id: L3-state + section: "3 — State" + claim: "Progress lives outside the model: steps taken, results, environment." + probe: {type: grep, pattern: "def record_iteration", paths: ["src/loopeng/memory/store.py"]} + + - id: L4-policy + section: "3 — Policy" + claim: "Something decides the next action from the current state." + probe: {type: grep, pattern: "class RefactorBrief|def build_brief", paths: ["src/loopeng/adapters/base.py", "src/loopeng/loop/refactor_brief.py"]} + + - id: L5-action-space + section: "3 — Action space" + claim: "The agent's available actions are declared, not open-ended." + probe: {type: grep, pattern: "class Factory|class Refiner|class Judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L6-observation + section: "3 — Observation" + claim: "Every action returns structured environment feedback." + probe: {type: grep, pattern: "class Verdict", paths: ["src/loopeng/adapters/base.py"]} + + - id: L7-evaluation + section: "3/10 — Evaluation is the controller" + claim: "A verifiable evaluator decides progress, not the maker's self-report." + probe: {type: grep, pattern: "def judge", paths: ["src/loopeng/adapters/base.py"]} + + - id: L8-maker-not-checker + section: "10 — evaluation must be trustworthy" + claim: "The thing that builds is not the thing that grades." + probe: {type: pytest, node: "tests/test_maker_checker.py"} + + - id: L9-reflection + section: "4/6 — Plan-Execute-Observe-Reflect, Self-Reflection" + claim: "Why the last attempt scored what it did is carried into the next attempt." + probe: {type: grep, pattern: "class ReflectionContext", paths: ["src/loopeng/adapters/base.py"]} + + - id: L10-replan-on-plateau + section: "4 — the plan is not immutable" + claim: "Feedback can force a change of strategy, not just another attempt." + probe: {type: grep, pattern: "plateau_pivots", paths: ["src/loopeng/config.py"]} + + - id: L11-retry-transient-only + section: "8 — Retry vs Recovery" + claim: "Only retryable (infrastructure) failures are retried; not every error." + probe: {type: grep, pattern: "last_infra_failure", paths: ["src/loopeng/adapters/base.py"]} + + - id: L12-recovery-state + section: "8 — Recovery keeps enough state to resume" + claim: "A failed change can be rolled back rather than restarting from zero." + probe: {type: pytest, node: "tests/test_checkpoint.py"} + + - id: L13-exit-success + section: "9 — explicit exits: success" + claim: "The loop stops when the goal is verifiably met." + probe: {type: grep, pattern: "CONVERGED", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L14-exit-budget + section: "9/14 — explicit exits: budget (iterations, tokens, wall clock)" + claim: "The loop stops on a spent budget, and the budget has more than one dimension." + probe: {type: grep, pattern: "ITERATION_CAP|TOKEN_CAP|WALL_CAP", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L15-exit-giveup + section: "9 — explicit exits: give up and report" + claim: "Repeated non-progress ends the run and reports failure instead of looping." + probe: {type: grep, pattern: "PLATEAU", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L16-safety-terminal + section: "13 — permission control" + claim: "A safety failure is terminal and unbypassable, whatever the score." + probe: {type: grep, pattern: "BLOCKED_SAFETY", paths: ["src/loopeng/loop/convergence.py"]} + + - id: L17-permission-boundary + section: "13 — which tools/data the agent may touch is limited" + claim: "Execution is jailed and shell metacharacters are refused." + probe: {type: grep, pattern: "within_workspace|shell=False", paths: ["src/loopeng/adapters/safety.py"]} + + - id: L18-human-in-the-loop + section: "12 — Human-in-the-Loop for high-risk actions" + claim: "High-risk completion requires a human, and the caller cannot self-approve." + probe: {type: grep, pattern: "class VerificationGate", paths: ["src/loopeng/config.py"]} + + - id: L19-hitl-unbypassable + section: "12 — the gate must actually hold" + claim: "An unattended run cannot pre-confirm its own result." + probe: {type: pytest, node: "tests/test_run_contract.py::test_contract_can_never_disable_the_human_gate"} + + - id: L20-multi-agent-graph + section: "11 — Loop becomes Graph with many agents" + claim: "Multiple agents are coordinated as a dependency graph, cycles refused." + probe: {type: grep, pattern: "Kahn|cycle", paths: ["src/loopeng/orchestration/coordinator.py"]} + + - id: L21-observability + section: "13 — every iteration leaves enough log/trace to debug" + claim: "A run is reconstructable after the fact from recorded evidence." + probe: {type: grep, pattern: "def render_report", paths: ["src/loopeng/autonomous/report.py"]} + + - id: L22-token-accounting + section: "14 — Token economics: measure tokens, time, iterations, success" + claim: "Token cost, wall time and iteration count are recorded per run." + probe: {type: grep, pattern: "token_cost", paths: ["src/loopeng/proof.py"]} + + - id: L23-cost-never-faked + section: "14 — measurement must be real to be useful" + claim: "An unavailable cost is omitted, never estimated into the record." + probe: {type: pytest, node: "tests/test_proof.py"} + + # ---- honest gap declarations: the rubric asserts these are ABSENT ---- + - id: L24-typed-failure-taxonomy + section: "8 — classify the error before choosing a response" + claim: "Failures are classified into a named taxonomy (referee-unavailable, adapter-contract, human-gate-timeout)." + expect: absent + probe: {type: grep, pattern: "class FailureType|REFEREE_UNAVAILABLE", paths: ["src/loopeng"]} + gap_note: "Only infra-vs-clean is distinguished. External eval item 3." + + - id: L25-branching-search + section: "7 — Tree of Thoughts" + claim: "The loop explores multiple candidate paths and prunes them." + expect: absent + probe: {type: grep, pattern: "tree_of_thought|beam_search", paths: ["src/loopeng"]} + gap_note: "Single-path refine with a dimension pivot. Deliberate: branching multiplies cost." + + - id: L26-trace-schema + section: "13 — observability as a first-class trace" + claim: "Runs emit a structured trace (spans/trace ids), not just a report." + expect: absent + probe: {type: grep, pattern: "opentelemetry|trace_id|def emit_span", paths: ["src/loopeng"]} + gap_note: "Reports and proof packs exist; a span-level trace schema does not." diff --git a/scripts/audit_loop_rubric.py b/scripts/audit_loop_rubric.py index 50721ef..82acf91 100644 --- a/scripts/audit_loop_rubric.py +++ b/scripts/audit_loop_rubric.py @@ -6,7 +6,8 @@ running system. This script is the missing half: every claim becomes a probe against shipped code or a real test run, and the score is whatever the probes return. -Discipline (playbook: operationalizing-a-paper-rubric-checklist-standard): +Discipline (the operationalizing-a-rubric playbook, which ships with the anyagent +skill rather than in this repo -- summarised here so this file stands alone): * Evidence is OBSERVED -- a symbol at a file:line, or a pytest node that actually passes. Never a claim in prose. @@ -98,8 +99,9 @@ def run_item(item: dict) -> dict: return {**item, "status": PASS if ok else FAIL, "evidence": evidence} -def audit() -> dict: - doc = yaml.safe_load(RUBRIC.read_text(encoding="utf-8")) +def audit(rubric: pathlib.Path | None = None) -> dict: + path = rubric or RUBRIC + doc = yaml.safe_load(path.read_text(encoding="utf-8")) results = [run_item(i) for i in doc["items"]] verifiable = [r for r in results if not r.get("expect")] @@ -113,6 +115,7 @@ def audit() -> dict: gate_ok = not missing and not unmeasured and all(g["status"] == PASS for g in gaps) return { + "rubric": str(path.relative_to(ROOT)), "source": doc["source"], "total_items": len(results), "verifiable": len(verifiable), @@ -129,7 +132,7 @@ def audit() -> dict: def render(a: dict) -> str: src = a["source"] L = ["# Agent Loop Engineering — conformance audit", "", - f"Rubric: `docs/rubrics/agent-loop-engineering.yml` — {a['total_items']} items drawn from " + f"Rubric: `{a['rubric']}` — {a['total_items']} items drawn from " f"*{src['title']}* ({src['publisher']}, received {src['received']}).", "", f"That source is a **{src['nature']}**. Every claim below is scored against shipped " "code or a test that was actually executed. No evidence means no.", "", @@ -169,12 +172,14 @@ def render(a: dict) -> str: def main() -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--rubric", default=None, + help="score an alternative rubric file (e.g. an archived version)") ap.add_argument("--json", action="store_true") ap.add_argument("--gate", action="store_true", help="exit 1 unless every item passes") ap.add_argument("--out") args = ap.parse_args() - a = audit() + a = audit(pathlib.Path(args.rubric).resolve() if args.rubric else None) text = json.dumps(a, indent=2) if args.json else render(a) if args.out: pathlib.Path(args.out).write_text(text + "\n", encoding="utf-8")