From 5f80c169e4f18346a77876804fc3e06eec6edeeb Mon Sep 17 00:00:00 2001 From: Sollan Systems Date: Thu, 9 Jul 2026 22:47:19 -0400 Subject: [PATCH] docs(self-eval): label terminal/repair/eval checks as documentation-completeness (#35) --- CONTRIBUTING.md | 13 +++++++++++-- README.md | 9 ++++++--- scripts/self_eval.py | 26 ++++++++++++++++++++------ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15fc7cd..5dfb995 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ you open a PR, all of these must be green (CI runs them on every push): ```bash # from the repo root python3 scripts/validate_frontmatter.py # SKILL.md frontmatter -python3 scripts/self_eval.py # 13 structural invariants +python3 scripts/self_eval.py # 13 structural/doc-completeness invariants python3 -m pytest -q scripts # the test suite python3 -m py_compile loop/*.py scripts/*.py python3 -m loop doctor examples/coverage-repair # quickstart still works @@ -20,6 +20,12 @@ python3 -m loop inspect examples/coverage-repair If you don't have the deps, prefix with `uv run --with pyyaml --with pytest`. +Most `self_eval.py` checks verify documentation-completeness — that the suite's +canonical vocabulary (terminal states, repair-record fields, eval layers + +metrics) is present in the skill prose — not that a running loop enforces it. The +runtime/behavioral gate is `python3 -m loop doctor` and the contract's own +`verify-*` scripts, which is why both appear in the list above. + ## Start here — the contributor funnel Every open starter issue names **the gate that proves the fix** — a @@ -53,7 +59,10 @@ Loop-Engineer-conformant contract v1. Drafts for the seeded issues live in ## Authoring or editing a skill -The `self_eval.py` checks encode hard rules — match them or the gate fails: +The `self_eval.py` checks encode hard rules — match them or the gate fails. Most +are documentation-completeness checks (the canonical vocabulary must appear in the +skill prose), not behavioral enforcement of a running loop — that gate is `loop +doctor` and the contract's own `verify-*` scripts: - **Frontmatter** must have `name:` and `description:`. The directory name **must equal** the frontmatter `name`. Keep the `description` a *quoted* YAML scalar (the suite quotes diff --git a/README.md b/README.md index 70a17f1..ee26367 100644 --- a/README.md +++ b/README.md @@ -389,9 +389,12 @@ claude plugin validate --strict .claude-plugin/plugin.json ``` The structural self-eval checks skill presence, frontmatter, cross-links, -terminal-state coverage, repair-record fields, eval metrics, templates, secret -patterns, dispatch examples, the bring-your-own-verifier default, the MIT -license, and README differentiation. +templates, secret patterns, dispatch examples, the bring-your-own-verifier +default, the MIT license, and README differentiation. Three further checks are +documentation-completeness scans: they confirm the canonical vocabulary for +terminal-state coverage, repair-record fields, and eval metrics is present in the +skill prose — not that a running loop enforces it (that runtime gate is `loop +doctor` and the contract's own `verify-*` scripts). --- diff --git a/scripts/self_eval.py b/scripts/self_eval.py index 368dde5..9237822 100644 --- a/scripts/self_eval.py +++ b/scripts/self_eval.py @@ -1,9 +1,13 @@ """Deterministic structural self-eval for the loop-engineer suite. Runs the 13 structural checks the suite grades itself by (the hard pass/fail -gate; the rubric in evals/rubric.md is the advisory layer above this). Reuses -``validate_frontmatter`` from this same scripts/ dir rather than reimplementing -the frontmatter parse. Expected structural facts live in +gate for suite structure; the rubric in evals/rubric.md is the advisory layer +above this). Most checks are documentation-completeness scans — they confirm the +canonical vocabulary (terminal states, repair-record fields, eval layers + +metrics) is present in the skill prose, not that a running loop enforces it. The +runtime/behavioral gate is ``loop doctor`` and the contract's own ``verify-*`` +scripts. Reuses ``validate_frontmatter`` from this same scripts/ dir rather than +reimplementing the frontmatter parse. Expected structural facts live in ``evals/cases/structural.json`` so the checks stay data-driven and in sync with the suite. @@ -182,6 +186,9 @@ def check_links_resolve(root, facts): return True, "all [[links]] resolve to a skill dir or known sibling" +# Documentation-completeness: confirms the canonical terminal-state vocabulary +# is present in loop-run's prose (a substring scan), not runtime enforcement that +# a loop actually reaches one of these states — that gate is loop doctor / verify-*. def check_terminal_states(root, facts): text = _read(root / "skills" / "loop-run" / "SKILL.md") missing = [s for s in facts["terminal_states"] if s not in text] @@ -190,6 +197,10 @@ def check_terminal_states(root, facts): return True, f"loop-run names all {len(facts['terminal_states'])} terminal states" +# Documentation-completeness for the field vocabulary: confirms the repair-record +# field names are present in loop-repair's prose (a substring scan). The second +# half IS structural — the examples/ failure_mode taxonomy scan parses example +# files and rejects any failure_mode value outside the canonical taxonomy. def check_repair_fields(root, facts): text = _read(root / "skills" / "loop-repair" / "SKILL.md") missing = [f for f in facts["repair_record_fields"] if f not in text] @@ -203,6 +214,9 @@ def check_repair_fields(root, facts): return True, f"loop-repair names all {n} repair fields; example failure_modes canonical" +# Documentation-completeness: confirms the 7 eval-layer names + 2 first-class +# metrics appear in loop-evals' prose (a normalized substring scan), not runtime +# enforcement that the eval harness actually runs those layers or computes metrics. def check_eval_layers_and_metrics(root, facts): ntext = _norm(_read(root / "skills" / "loop-evals" / "SKILL.md")) wanted = list(facts["eval_layer_names"]) + list(facts["first_class_metrics"]) @@ -324,9 +338,9 @@ def check_byo_default(root, facts): ("frontmatter-valid", check_frontmatter_valid), ("references-used", check_references_used), ("links-resolve", check_links_resolve), - ("terminal-states-complete", check_terminal_states), - ("repair-record-fields", check_repair_fields), - ("eval-layers-and-metrics", check_eval_layers_and_metrics), + ("terminal-states-documented", check_terminal_states), + ("repair-record-fields-documented", check_repair_fields), + ("eval-layers-and-metrics-documented", check_eval_layers_and_metrics), ("templates-present", check_templates_present), ("no-secrets", check_no_secrets), ("dispatch-names-model", check_dispatch_names_model),