diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 528e2fa..8277c08 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -4,25 +4,13 @@ "name": "QuantEcon" }, "description": "QuantEcon's shared Claude Code skills and tools", - "version": "0.2.0", + "version": "0.3.0", "plugins": [ { "name": "qe", "source": "./qe", - "version": "0.6.0", - "description": "QuantEcon's author-facing base skills — working through a PR's review feedback, turning audit reports into tracked work projects, and creating, reading and carrying work-plan state across agent sessions" - }, - { - "name": "benchmark", - "source": "./benchmark", - "version": "0.4.0", - "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code" - }, - { - "name": "audit", - "source": "./audit", - "version": "0.2.0", - "description": "Bulk, read-only audits of a QuantEcon repository — issue triage, PR review, technical debt, translation parity — each producing an evidence-cited report bundle" + "version": "0.7.0", + "description": "QuantEcon's shared agent skills under one namespace — working through a PR's review feedback, advising on and scoring lecture acceleration (benchmark), bulk read-only repository audits, turning audit reports into tracked work projects, and creating, reading and carrying work-plan state across agent sessions" } ] } diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 22c01be..5c9ae0b 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -45,7 +45,7 @@ jobs: BASE_REF: ${{ github.base_ref }} run: python scripts/check-version-bump.py --base "origin/$BASE_REF" - # The benchmark plugin's claim is that no score is ever written by hand: + # The benchmark skill's claim is that no score is ever written by hand: # every scorecard is a deterministic function of its evidence.json. That # only stays true if it is checked. A non-empty diff here means either a # scorecard was hand-edited, or a rubric change moved a published verdict @@ -53,13 +53,13 @@ jobs: # to re-run these two commands and commit, so the verdict move lands in # the diff where a reviewer sees it. Stdlib only; no install step needed. - name: Scorecards reproduce from evidence - working-directory: benchmark + working-directory: qe run: | - python scripts/scoring/score.py references/examples/ge_arrow - python scripts/scoring/score.py references/examples/markov_asset - python scripts/scoring/score.py references/fixtures/rubric_v2 - git diff --exit-code -- 'references/examples/*/results/scorecard.json' \ - 'references/fixtures/*/results/scorecard.json' + python scripts/benchmark/scoring/score.py references/benchmark/examples/ge_arrow + python scripts/benchmark/scoring/score.py references/benchmark/examples/markov_asset + python scripts/benchmark/scoring/score.py references/benchmark/fixtures/rubric_v2 + git diff --exit-code -- 'references/benchmark/examples/*/results/scorecard.json' \ + 'references/benchmark/fixtures/*/results/scorecard.json' # A separate job on purpose: this one installs an ~85 MB npm toolchain, and # running it beside `validate` rather than inside it keeps that job four fast diff --git a/AGENTS.md b/AGENTS.md index 7d2fe6f..9148409 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Guidance for AI coding agents and human contributors working in `QuantEcon/skill ## What this repository is -A [Claude Code plugin marketplace](https://code.claude.com/docs/en/plugin-marketplaces) holding QuantEcon's shared agent skills and the deterministic scripts they drive. The goal is to share institutional knowledge — the checks, rubrics and procedures experienced maintainers already apply by hand — so the same work produces consistent results wherever it runs. Orientation is in [README.md](README.md); what has actually shipped is in [CATALOG.md](CATALOG.md); work in flight is in the per-plugin tracking issues. +A [Claude Code plugin marketplace](https://code.claude.com/docs/en/plugin-marketplaces) holding QuantEcon's shared agent skills and the deterministic scripts they drive. The goal is to share institutional knowledge — the checks, rubrics and procedures experienced maintainers already apply by hand — so the same work produces consistent results wherever it runs. Orientation is in [README.md](README.md); what has actually shipped is in [CATALOG.md](CATALOG.md); work in flight is in the per-family tracking issues. **The repo is early, and its conventions are deliberately loose.** Where a doc describes a report shape, a phase division, a naming form or a directory layout, read it as what an existing skill does rather than as a contract a new one must satisfy — see [CATALOG.md § Principles](CATALOG.md#principles). The few things that genuinely must hold are stated plainly, with their reasons, and they are all about keeping a skill's output checkable by someone who will not re-run it. @@ -20,7 +20,7 @@ What this means in practice: - **Skills point outward.** When a skill needs a rule, a convention, or a procedure that already exists in the [QuantEcon manual](https://manual.quantecon.org) or in `QuantEcon/style-guide`, it cites and links to it. A `SKILL.md` carries only what the skill itself adds: the procedure it runs, the judgement it applies, the output it produces. - **Rule text is authored upstream only.** Style-rule text lives in `QuantEcon/style-guide`; when the style skills land ([#3](https://github.com/QuantEcon/skills/issues/3)) this repo will carry a rendered, drift-checked snapshot of it — never hand-authored here. -- **Numbers drift fastest** — weights, thresholds, verdict bands, versions. The rubric's weights are stated in [`benchmark/references/EVALUATION_FRAMEWORK.md`](benchmark/references/EVALUATION_FRAMEWORK.md) and implemented once in `benchmark/scripts/scoring/rubric.py`; anywhere else they come up, quote with a pointer rather than re-tabulating. +- **Numbers drift fastest** — weights, thresholds, verdict bands, versions. The rubric's weights are stated in [`qe/references/benchmark/EVALUATION_FRAMEWORK.md`](qe/references/benchmark/EVALUATION_FRAMEWORK.md) and implemented once in `qe/scripts/benchmark/scoring/rubric.py`; anywhere else they come up, quote with a pointer rather than re-tabulating. - **Every topic has an owning doc** (see the map below). Before adding a section, work out which file owns the topic, put it there, and link from wherever else it comes up. - **Across boundaries, link — don't copy.** An installed plugin ships only its own directory, so a reference to a repo-level file or another plugin is an absolute GitHub URL, never a duplicated paragraph ([developing-skills § Conventions](docs/developing-skills.md#conventions)). - **The one deliberate exception**: a `SKILL.md` frontmatter `description` must stand alone, because it is what natural-language invocation matches against. Restate what the skill does in that one sentence; the details stay behind the link. @@ -34,15 +34,15 @@ Before adding a paragraph, check whether it already exists. If it does, link to | What the marketplace is, installation (local, lecture repos, CI) | [README.md](README.md) | | Using the skills: setup, invocation, what to expect | [docs/using-skills.md](docs/using-skills.md) | | Contributing: layout, conventions, dev loop, local testing, versioning, releases and tagging, PR flow | [docs/developing-skills.md](docs/developing-skills.md) | -| What changed in a plugin, release by release | `/CHANGELOG.md` — one per plugin, since the plugin is the released artifact and ships only its own directory | +| What changed, release by release | [qe/CHANGELOG.md](qe/CHANGELOG.md) — inside the plugin, since the plugin is the released artifact and ships only its own directory; the retired benchmark/audit streams are historical sections there | | Running an evaluation by hand, end to end | [docs/tutorial-run-an-evaluation.md](docs/tutorial-run-an-evaluation.md) | | Running a whole-tracker audit, and reviewing what it produces | [docs/tutorial-run-an-audit.md](docs/tutorial-run-an-audit.md) | -| The benchmark skill: modes, report format, manual pipeline | [benchmark/README.md](benchmark/README.md) | -| Rubric: dimensions, weights, anchors, verdict bands | [benchmark/references/EVALUATION_FRAMEWORK.md](benchmark/references/EVALUATION_FRAMEWORK.md) | +| The benchmark skill: modes, report format, manual pipeline | [qe/references/benchmark/README.md](qe/references/benchmark/README.md) | +| Rubric: dimensions, weights, anchors, verdict bands | [qe/references/benchmark/EVALUATION_FRAMEWORK.md](qe/references/benchmark/EVALUATION_FRAMEWORK.md) | | Style rule text and schema | `QuantEcon/style-guide` (upstream — never authored in this repo) | | What has shipped, and the principles behind it | [CATALOG.md](CATALOG.md) | | Parked ideas, not committed to | [Low-priority enhancement issues](https://github.com/QuantEcon/skills/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement+label%3Alow-priority) in the tracker, one per idea with its merit assessment | -| Work in flight, per plugin | issues [#3](https://github.com/QuantEcon/skills/issues/3) (`qe`), [#4](https://github.com/QuantEcon/skills/issues/4) (`benchmark`), [#12](https://github.com/QuantEcon/skills/issues/12) (`audit`) | +| Work in flight, per skill family | issues [#3](https://github.com/QuantEcon/skills/issues/3) (style), [#4](https://github.com/QuantEcon/skills/issues/4) (benchmark), [#12](https://github.com/QuantEcon/skills/issues/12) (audit) | ## Working in this repo diff --git a/CATALOG.md b/CATALOG.md index df599c4..e98f67c 100644 --- a/CATALOG.md +++ b/CATALOG.md @@ -1,14 +1,16 @@ # Skill catalog -What this marketplace installs today, filtered to what is **operational**: a skill appears here once it is merged *and* runs to its stated purpose, not merely once its files are on `main`. Listing is not a claim about how far a skill has been validated — the State column answers that for each one, and says so plainly when the answer is "not much". Anything not listed does not exist yet as a usable skill, however firmly it has been discussed. Work in flight lives in the tracking issue for its plugin; ideas nobody has committed to live in the tracker as [low-priority enhancement issues](https://github.com/QuantEcon/skills/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement+label%3Alow-priority). +What this marketplace installs today, filtered to what is **operational**: a skill appears here once it is merged *and* runs to its stated purpose, not merely once its files are on `main`. Listing is not a claim about how far a skill has been validated — the State column answers that for each one, and says so plainly when the answer is "not much". Anything not listed does not exist yet as a usable skill, however firmly it has been discussed. Work in flight lives in the tracking issue for its skill family; ideas nobody has committed to live in the tracker as [low-priority enhancement issues](https://github.com/QuantEcon/skills/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement+label%3Alow-priority). -Scaffolding is deliberately absent — and as of `qe` 0.6.0 it no longer ships at all: an unbuilt skill lives only as the plan in its plugin's tracking issue, so installing a plugin never puts a menu entry in front of you that does nothing. (Through 0.5.0 the style skills shipped as stubs reporting "not yet operational"; that practice is retired.) +Scaffolding is deliberately absent — and as of `qe` 0.6.0 it no longer ships at all: an unbuilt skill lives only as the plan in its family's tracking issue, so installing the plugin never puts a menu entry in front of you that does nothing. (Through 0.5.0 the style skills shipped as stubs reporting "not yet operational"; that practice is retired.) -| Plugin | Skills | State | Tracking | +Everything ships in the single `qe` plugin ([#43](https://github.com/QuantEcon/skills/issues/43), qe 0.7.0); the families keep their own tracking issues. + +| Family | Skills | State | Tracking | |---|---|---|---| -| **`qe`** — author-facing skills, from drafting a lecture to merging its PR | `/qe:copilot-review` | Operational, and validated from an installed plugin on 2026-08-03: plugin-root path resolution, cross-repo mode, and running from outside a working tree. The style skills (`check-style` and six per-category siblings) shipped as scaffolding through 0.5.0 and were removed in 0.6.0; their plan lives in [#3](https://github.com/QuantEcon/skills/issues/3), and they return once the rule snapshot and deterministic preflight land. The `workplan-*` family — `/qe:workplan-project` (0.3.0 as `/qe:workplan`, renamed 0.4.0; report bundle → tracking issue with sub-issues) and `/qe:workplan` (0.5.0, consolidating 0.4.0's `workplan-issue` and `workplan-update` and adding a read verb; the work-plan issue's whole lifecycle — create, read/validate/recommend, resume, update, close-and-succeed) — is merged as complete procedures with no validated runs yet, so it is likewise not listed until it has one. Note the name reuse: `/qe:workplan` meant the report-to-project skill for one morning in 0.3.0; from 0.5.0 it is the lifecycle skill. | [#3](https://github.com/QuantEcon/skills/issues/3) | -| **`benchmark`** — evaluating accelerated lecture implementations | `/benchmark:review-acceleration` | Operational for workspace runs: rubric v2, a deterministic scoring engine, and two complete worked evaluations as regression baselines. | [#4](https://github.com/QuantEcon/skills/issues/4) | -| **`audit`** — bulk, read-only repository audits | `/audit:issues` | Run once **as a skill** — a 230-item tracker on 2026-07-28, 22 minutes, seven plugin defects found and recorded ([record](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md)). Both severity-1 defects are fixed as of 0.2.0 ([#34](https://github.com/QuantEcon/skills/pull/34)): a `[verified]` citation must now resolve on the ref the audit named, and phase 2 checkpoints its closed pass as well as its open one. That validates the method as far as one run goes and no further: its central claim, resumability, is still untested, because the run was never interrupted. Further runs: [#16](https://github.com/QuantEcon/skills/issues/16). | [#12](https://github.com/QuantEcon/skills/issues/12), [#16](https://github.com/QuantEcon/skills/issues/16) | +| **Author-facing** — from drafting a lecture to merging its PR | `/qe:copilot-review` | Operational, and validated from an installed plugin on 2026-08-03: plugin-root path resolution, cross-repo mode, and running from outside a working tree. The style skills (`check-style` and six per-category siblings) shipped as scaffolding through 0.5.0 and were removed in 0.6.0; their plan lives in [#3](https://github.com/QuantEcon/skills/issues/3), and they return once the rule snapshot and deterministic preflight land. The `workplan-*` family — `/qe:workplan-project` (0.3.0 as `/qe:workplan`, renamed 0.4.0; report bundle → tracking issue with sub-issues) and `/qe:workplan` (0.5.0, consolidating 0.4.0's `workplan-issue` and `workplan-update` and adding a read verb; the work-plan issue's whole lifecycle — create, read/validate/recommend, resume, update, close-and-succeed) — is merged as complete procedures with no validated runs yet, so it is likewise not listed until it has one. Note the name reuse: `/qe:workplan` meant the report-to-project skill for one morning in 0.3.0; from 0.5.0 it is the lifecycle skill. | [#3](https://github.com/QuantEcon/skills/issues/3) | +| **Benchmark** — evaluating accelerated lecture implementations | `/qe:benchmark` (until qe 0.7.0: `/benchmark:review-acceleration`) | Operational for workspace runs: rubric v2, a deterministic scoring engine, and two complete worked evaluations as regression baselines. | [#4](https://github.com/QuantEcon/skills/issues/4) | +| **Audit** — bulk, read-only repository audits | `/qe:audit-issues` (until qe 0.7.0: `/audit:issues`) | Run once **as a skill** — a 230-item tracker on 2026-07-28, 22 minutes, seven plugin defects found and recorded ([record](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md)). Both severity-1 defects are fixed as of audit 0.2.0 ([#34](https://github.com/QuantEcon/skills/pull/34)): a `[verified]` citation must now resolve on the ref the audit named, and phase 2 checkpoints its closed pass as well as its open one. That validates the method as far as one run goes and no further: its central claim, resumability, is still untested, because the run was never interrupted. Further runs: [#16](https://github.com/QuantEcon/skills/issues/16). | [#12](https://github.com/QuantEcon/skills/issues/12), [#16](https://github.com/QuantEcon/skills/issues/16) | Installation and setup are in [README.md](README.md); what it is like to run one is in [docs/using-skills.md](docs/using-skills.md). @@ -18,7 +20,7 @@ The point of the marketplace is to **share institutional knowledge** — the che - **Few, high-frequency skills** over many niche ones, each validated against actual PR history. The 2026-07-21 analysis of ~630 merged PRs across the four main lecture repos is the evidence base: style was the largest recurring theme by a wide margin, which is why it is the flagship. A skill justified by breadth rather than frequency, as the audit family is, should say so. - **Report first, fix on request.** Skills produce a structured report and offer fixes; they never silently edit. Safe to run in CI, and authors stay in control. -- **Cited claims; computed scores.** Every finding carries a citation — a rule ID plus `file:line`, or a number plus its source. Skills whose output is a findings list need nothing more. Skills that aggregate judgements into a scored verdict use the evidence-file pattern from the benchmark plugin: judgement recorded as cited answers, every score computed by a deterministic engine, never typed by hand (see [docs/developing-skills.md](docs/developing-skills.md)). -- **Scaffolding is advice, not instruction.** Report shapes, phase divisions, naming forms and directory conventions are described as what an existing skill does, not as contracts a new one has to satisfy. Three plugins is not enough to know which of them generalise, and a rule invented from one worked example mostly succeeds at forcing the next skill into the first one's shape. A skill can be a single `SKILL.md`. Where something genuinely must hold — read-only boundaries, cited claims, a stated coverage of what was and was not checked — say so plainly and give the reason; everything else can converge later, once there is something to generalise from. +- **Cited claims; computed scores.** Every finding carries a citation — a rule ID plus `file:line`, or a number plus its source. Skills whose output is a findings list need nothing more. Skills that aggregate judgements into a scored verdict use the evidence-file pattern from the benchmark skill: judgement recorded as cited answers, every score computed by a deterministic engine, never typed by hand (see [docs/developing-skills.md](docs/developing-skills.md)). +- **Scaffolding is advice, not instruction.** Report shapes, phase divisions, naming forms and directory conventions are described as what an existing skill does, not as contracts a new one has to satisfy. Three skill families is not enough to know which of them generalise, and a rule invented from one worked example mostly succeeds at forcing the next skill into the first one's shape. A skill can be a single `SKILL.md`. Where something genuinely must hold — read-only boundaries, cited claims, a stated coverage of what was and was not checked — say so plainly and give the reason; everything else can converge later, once there is something to generalise from. Note what the last two have in common: the rules stated firmly are the ones that keep output *checkable by someone who will not re-run it*. That is the test worth applying before writing any new rule down. diff --git a/README.md b/README.md index c5c0977..a5121a5 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,21 @@ Each plugin bundles one area of work — a skill (the instructions Claude follow 📖 **[quantecon.github.io/skills](https://quantecon.github.io/skills)** — the documentation, rendered and navigable. It is built from the files in this repository, so reading either one gets you the same content. -## Plugins +## The `qe` plugin -| Plugin | For | Covers | +One plugin, one namespace ([#43](https://github.com/QuantEcon/skills/issues/43)): every skill installs as `/qe:`, so an invocation always reads as a QuantEcon skill. + +| Skill | For | Does | |---|---|---| -| `qe` | Authors and RAs writing lectures; maintainers organising work | Working through the review feedback on a PR once it is open, and the `workplan-*` family — turning audit reports into tracked work projects and carrying work-plan state between agent sessions. Style checks against the QuantEcon style guide are planned ([skills#3](https://github.com/QuantEcon/skills/issues/3)) | -| `benchmark` | Maintainers reviewing accelerated implementations | Measured, rubric-scored evaluation of a conversion | -| `audit` | Maintainers sweeping a whole repository | Bulk, read-only audits — every issue, every PR, a codebase, a translated series — each producing a written report | +| `/qe:copilot-review` | Authors with an open PR | Works through Copilot's review comment by comment: verdict and fix per comment, threaded replies so each resolves from the GitHub UI | +| `/qe:benchmark` | Maintainers weighing acceleration | Advises whether a lecture is worth converting at all (triage), or scores a submitted NumPy→JAX/Numba conversion against the rubric (review) | +| `/qe:audit-issues` | Maintainers sweeping a repository | Whole-tracker audit, read-only: every issue's status verified against the code, tiered into the repo's plan, delivered as a report bundle | +| `/qe:workplan-project` | Maintainers organising work | Turns an audit or review report into a tracking issue with linked sub-issues | +| `/qe:workplan` | Anyone carrying work across sessions | The work-plan issue's whole lifecycle — create, read (validate and recommend, writing nothing), resume, update, close-and-succeed | -`qe` is the author-facing surface — one memorable prefix for everyday work, spanning a lecture's life from drafting to merge and the work planning around it. `/qe:copilot-review` picks the lecture up after the PR is open, working through Copilot's review comment by comment; the style-check family (`/qe:check-style` against the QuantEcon style guide) is planned in [skills#3](https://github.com/QuantEcon/skills/issues/3) and lands once its rule snapshot and deterministic preflight exist. The `workplan-*` family serves the maintainer end with two skills: `/qe:workplan-project` turns an audit or review report into a tracking issue with sub-issues, and `/qe:workplan` carries the single work-plan issue that holds state between agent sessions through its whole lifecycle — create, read (validate against live state and recommend next steps, writing nothing), resume, update, and close-and-succeed. `benchmark` and `audit` are specialist toolkits, installed by the maintainers who need them. +Style checks against the QuantEcon style guide (`/qe:check-style`) are planned in [skills#3](https://github.com/QuantEcon/skills/issues/3) and land once their rule snapshot and deterministic preflight exist. Until qe 0.7.0 the benchmark and audit skills were the separate `benchmark` and `audit` plugins — if you installed those, uninstall them (`claude plugin uninstall benchmark@quantecon audit@quantecon`) so the retired names don't linger in your menu. -**Which skills work right now is in [CATALOG.md](CATALOG.md)** — it lists what has merged *and* is operational, so this page does not repeat it. Skills not yet built live only as plans in their plugin's tracking issue — nothing ships as a non-working menu entry. Ideas nobody has committed to are tracked as [low-priority enhancement issues](https://github.com/QuantEcon/skills/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement+label%3Alow-priority). +**How far each skill has been validated is in [CATALOG.md](CATALOG.md)** — it lists what has merged *and* is operational, so this page does not repeat it. Skills not yet built live only as plans in the tracking issues — nothing ships as a non-working menu entry. Ideas nobody has committed to are tracked as [low-priority enhancement issues](https://github.com/QuantEcon/skills/issues?q=is%3Aissue+is%3Aopen+label%3Aenhancement+label%3Alow-priority). ## Documentation @@ -36,21 +40,18 @@ Lecture repositories opt in by checking the following into their `.claude/settin } }, "enabledPlugins": { - "qe@quantecon": true, - "benchmark@quantecon": true + "qe@quantecon": true } } ``` -`audit` is deliberately absent from the lecture-repo block: it is maintainer tooling, and the plugin is the enable unit, so auto-installing it would put org-wide audit skills in every author's command list. Maintainers install it themselves. +Since the plugin is the enable unit and there is now one plugin, every consumer gets the full skill list — including the read-only maintainer tooling (audits, benchmark). That trade was weighed in [#43](https://github.com/QuantEcon/skills/issues/43): two extra read-only entries in a five-item menu, against a namespace every invocation shares. ### Manual (any project) ``` /plugin marketplace add QuantEcon/skills /plugin install qe@quantecon -/plugin install benchmark@quantecon -/plugin install audit@quantecon ``` Those are slash commands in a Claude Code session; **restart the session afterwards**, since plugins register at startup. `/plugin` is a terminal-CLI built-in, so in the VS Code extension or the web app use the equivalent `claude plugin …` CLI commands instead — see [using-skills § Setup](docs/using-skills.md#setup). @@ -63,8 +64,8 @@ The official action accepts the marketplace and plugin directly: - uses: anthropics/claude-code-action@v1 with: plugin_marketplaces: "https://github.com/QuantEcon/skills.git" - plugins: "benchmark@quantecon" - prompt: "/benchmark:review-acceleration " + plugins: "qe@quantecon" + prompt: "/qe:benchmark " ``` ## Contributing diff --git a/audit/.claude-plugin/plugin.json b/audit/.claude-plugin/plugin.json deleted file mode 100644 index 8d0b6d4..0000000 --- a/audit/.claude-plugin/plugin.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "audit", - "description": "Bulk, read-only audits of a QuantEcon repository — issue triage, PR review, technical debt, translation parity — each producing an evidence-cited report bundle", - "version": "0.2.0", - "author": { "name": "QuantEcon" } -} diff --git a/audit/CHANGELOG.md b/audit/CHANGELOG.md deleted file mode 100644 index d94342b..0000000 --- a/audit/CHANGELOG.md +++ /dev/null @@ -1,90 +0,0 @@ -# Changelog — `audit` - -Every released version of this plugin, newest first. A version exists because the content below shipped in it: the plugin cache is keyed by version string, so what you have installed is exactly the entries down to the version `claude plugin list` reports for `audit`. - -Versions are [semver](https://semver.org) as a user of this plugin experiences it — a new skill, or a procedure that now does something materially different, is a minor bump; a correction that leaves the procedure as it was is a patch. Nothing below 1.0.0 promises stability. - -Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/audit)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). - -## 0.2.0 — 2026-08-07 - -The two severity-1 defects from the first measured run, which are the same defect at different altitudes: an audit's own record claiming more than it can support. - -**Changed** - -- **`[verified]` now requires evidence reachable from the ref the audit named.** Doctrine §2 is the single statement of it — §1 rule 1 no longer carries its own copy of the accepted-forms list, which is how the two drifted apart in the first place — and it covers every citation form — a commit must be an ancestor of the baseline ref, a `file:line` must be that line *on the ref* rather than in the working tree, a PR must be merged into it — and `/audit:issues` runs `git merge-base --is-ancestor ` before tagging a commit citation. Run 1's headline finding cited a commit that is real, does touch the file, and exists only on an unmerged branch, while the report's header said it had verified against `main`. A citation that resolves for its author and not for its reader is worse than an untagged claim, because the tag is what invited the trust. Evidence that genuinely lives off-ref stays citable — as the open PR it is, tagged `[stated]` or `[inferred]`. -- **Phase 2 checkpoints both of its passes.** `findings.md` now carries `## Open` and `## Closed` sections, and the resume rule partitions `issues.json` by state and resumes each side independently at the lowest number with no entry. Run 1 wrote only the 56 open issues to the checkpoint and sent the 62 closed ones straight to the catalog, so a run interrupted during the closed pass would have re-verified all 62 from scratch while reporting itself complete — the single-block resume rule shipped in 0.1.2 could not see the difference. - -## 0.1.4 — 2026-08-03 - -Doctrine §4's rule survived the first measured run; its justification did not. The section is re-derived from what that run actually produced, and the cost figures the skill quotes are replaced with measured ones. - -**Changed** - -- Doctrine §4 is renamed from "Surviving a long run" to "Checkpointing", and rests on three reasons that hold at any duration rather than on the claim that audits outlive sessions. The first measured run refuted that claim outright — 230 items in 22 minutes, with no context exhaustion, rate limit or sleeping machine in play. The strongest replacement reason is checkable: the per-item log is what the final enumeration is assembled *from*, and what a reviewer counts the coverage numbers against. -- `/audit:issues` no longer describes itself as "long-running by design — a hundred-issue repo is a multi-hour run". It now quotes the measured cost: roughly **10 seconds per open issue**, with a 230-item tracker carrying 56 open issues taking 22 minutes. Cost tracks open issues needing verification rather than total items, so a large tracker with a small open set is cheaper than a small one with a large set. -- The cost figures are stated so the two measures cannot be confused. Previously a reader met "roughly 10 seconds per open issue" beside "a 230-item tracker with 56 open took 22 minutes" and could not reconcile them — 56 × 10 s is 9 minutes, not 22. The 10-second rate is phase 2 alone; 22 minutes is end to end, and the remaining phases are largely fixed. Both numbers now say which question they answer. -- Checkpoint artifacts are named where they carry evidence rather than at every phase boundary out of symmetry — a checkpoint written and superseded minutes later without ever being read earns nothing. - -## 0.1.3 — 2026-08-03 - -**Added** - -- This changelog. - -**Fixed** - -- `/audit:issues`'s frontmatter `description` was an unquoted YAML plain scalar containing `Read-only: it recommends…`. A `: ` inside a plain scalar is a parse error, so a strict loader drops the skill's metadata rather than reading it, and `claude plugin validate` rejects the file outright. The value is now quoted. Nothing about the procedure changed. - -## 0.1.2 — 2026-07-28 - -Resolves the contradiction that told an audit to write its bundle into the repo it promised not to touch: the boundary is now mutation, not writing, and the skill says exactly where to put its working directory so a run leaves `git status` clean. - -**Added** - -- Discovery-ordered working-directory selection, taken from contact with a real repo: prefer a location the repo already ignores (`.dev/scratch/audit-/` in QuantEcon repos, where `.dev/scratch/*` is already gitignored), fall back to an untracked `.audit/-/` at the checkout root, then to somewhere outside the checkout entirely. Which one was used goes in the report's method section. -- Doctrine §3 now says explicitly that a run may write its own working directory, including inside the audited checkout — provided the directory stays untracked and nothing is added to `.gitignore`, since that would itself be an edit to a tracked file. - -**Changed** - -- Doctrine §3 narrowed from "no branch or file changes in the audited repo" to what it was always protecting — content and history: no commits, no pushes, no branches, no edits to tracked files. Mutation, not writing, is the boundary. -- `deliverables.md` states the split: writing the bundle is the audit's job, committing or publishing it is a human step taken after reading it. - -**Fixed** - -- The read-only/working-directory contradiction the plugin carried since 0.1.0 — §3 forbade file changes in the audited repo while `deliverables.md` made that repo's own notes system the bundle's first-choice destination, and 0.1.1's default `--out` wrote there too. A run following the docs literally could not satisfy both. - -## 0.1.1 — 2026-07-28 - -An interrupted run can actually be resumed: the intermediate artifacts now have names and locations, phase 2 appends per item instead of writing at the end, and the bundle shrinks to fit a small tracker. - -**Added** - -- A stated working-directory layout under `--out` (`.audit/-/` by convention): `snapshot/` from phase 1, `findings.md` from phase 2, `links.md` from phase 3, and the delivered `01-…`/`02-…`/`03-…`/`README.md` bundle from phase 4. Previously phases 2 and 3 produced "per-item findings" and "the cluster map" with no filename and no location, so resuming worked only if two sessions independently invented the same file. -- A stated resume rule: on restart, read `findings.md` and resume at the lowest number in `issues.json` with no entry, re-verifying the last entry rather than trusting a possibly truncated write. -- `meta.json` records `fetched_by`, the account the snapshot was taken as — which matters because visibility on the org's private repos is per-account. - -**Changed** - -- Phase 2 appends each item's finding to `findings.md` as it is verified, in the catalog entry format, so phase 4 assembles the catalog instead of re-deriving it. -- The bundle scales to the tracker: below roughly 30 open issues, fold the catalog and the link graph into the report, keep the `README.md` index, and say which shape was used in the coverage statement. Four unconditional documents forced three files of padding on a small tracker, and padding makes a report less checkable. -- Doctrine §4 now states the general rule: a checkpoint owes a findable name and incremental writes, or it is a claim about resumability rather than the property itself. - -**Fixed** - -- `meta["authenticated"]` is removed, not deprecated. It could only ever be `true` (preflight exits on every unauthenticated path), so it was a provenance field carrying no evidence — in the plugin whose doctrine is that every claim carries its evidence class. **Anything reading that field must switch to `fetched_by`.** -- An interrupted phase 2 now loses one item rather than the whole phase — it was the phase specified to write on completion, and the phase a hundred-item run dies inside rather than between. - -## 0.1.0 — 2026-07-27 - -First release. `/audit:issues` sweeps an entire GitHub tracker — open and closed — verifies each item against the code rather than the thread, tiers the open set into the repo's existing plan, and delivers an evidence-cited report bundle, without ever touching the tracker. - -- `/audit:issues ` — a whole-tracker audit in five phases (snapshot, per-item verification, cross-link graph, tiered report, coverage self-audit). The four runbook fields (plan anchor, tier scheme, repo type, notes system) are optional arguments with documented discovery, so the usual invocation is just the repo. -- A deterministic snapshot step, `scripts/fetch_tracker.py OWNER/REPO --out `: every issue and PR in any state with full comment threads (and PR reviews, and `closingIssuesReferences`) in two `gh` round trips, written as `meta.json`, `issues.json`, `prs.json`, `coverage.json`. Closed threads cost nothing extra to read, and the snapshot freezes the audit's point in time so "events after the snapshot" is a stated property of the report instead of an unnoticed gap. -- `coverage.json` reconciliation: captured items against the number sequence `1..max`, discussion counts split open/closed, and an explicit truncation flag when a stream returns exactly at `--limit` (default 1000) — a case indistinguishable from truncation, so it is surfaced rather than swallowed. PR review bodies count toward captured discussion, not just comments: on the example repo, closed PRs carried 374 reviews against 28 comments. -- Snapshot files are written in issue/PR number order, so two runs over an unchanged tracker are byte-identical and a re-fetch diffs down to what actually changed. -- Thread payloads are shape-asserted at capture, so a `gh` build returning counts instead of lists fails by name at the point of capture rather than crashing later or silently under-reporting threads while the report still claims thread-completeness. -- Preflight that refuses to start without `gh` and an authenticated account, because the anonymous API is 60 req/h per IP and returns nothing at all for the org's private repos. -- Plugin-level method shared by every future audit skill: `references/doctrine.md` (trust rules, evidence classes `[verified]`/`[stated]`/`[inferred]`, the read-only boundary, checkpointing, the coverage self-audit), `references/quantecon-context.md` (repo types, label ownership, the cross-repo graph, access, and the caveat that an HTML-reconstructed thread may start mid-conversation), and `references/deliverables.md` (what an audit owes its reader, and where a bundle may land). -- QuantEcon-specific triage judgement: tier by repo type (a build break in a lecture repo and a consumer-visible change in an action repo outrank thread activity), check sibling repos before concluding, leave label application to `qe`, and keep GitHub closing keywords out of drafted cross-repo references so drafted text cannot close an upstream item when someone posts it. -- The four-document bundle, the five phases and "produces a bundle" are stated as a worked example rather than a requirement, after a single execution. What an audit owes its reader — coverage statement, evidence tag per claim, recommendations marked as proposals, drafted comments marked unsent, a date and a named snapshot — stays mandatory and presumes no file count. diff --git a/audit/README.md b/audit/README.md deleted file mode 100644 index d3ab849..0000000 --- a/audit/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# audit - -Bulk audits of a QuantEcon repository. Where `qe` serves an author working on one lecture and `benchmark` evaluates one conversion, these skills sweep a whole portfolio — every issue, every PR, a whole codebase, a whole translated series — and deliver a report. - -## Skills - -| Skill | Audits | Status | -|---|---|---| -| [`/audit:issues`](skills/issues/SKILL.md) | Every issue, open and closed: status verified against the code, tiered into the repo's plan | runbook landed | -| `/audit:prs` | Every open PR: does it solve a real issue, is it mergeable, what should the review say | candidate | -| `/audit:tech-debt` | A codebase's accumulated debt, with a filing-ready issue catalog | candidate | -| `/audit:translations` | Parity between a source series and a translation (`lecture-python.myst` ↔ `lecture-python.zh-cn`) | candidate | - -Only the first is written. The rest are candidates, tracked in [issue #12](https://github.com/QuantEcon/skills/issues/12) — each still needs the evidence a skill here normally carries before anyone writes it. Shipping one first is the point: the shared method gets proven against a real procedure before more are built on top of it. - -## What belongs here - -Two tests: - -1. **Bulk** — it sweeps a portfolio, not an item. Reviewing one PR's technical quality is not an audit; reviewing all of them is. -2. **Read-only** — it observes and reports. No skill here mutates a tracker, a branch, or a file in the audited repo ([doctrine §3](references/doctrine.md#3-read-only-boundary)). - -The read/write line is the one that matters, and it is deliberate: it mirrors the org's own automation split, where the family boundary *is* the permission boundary. Anything that acts on findings — filing the catalog as issues, posting the drafted comments, applying labels — is a separate human-invoked step, which is what makes this family safe to point at any repo and safe to run headlessly. - -An audit also produces a written report rather than a chat answer, since the point is something a reader can check later. That says nothing about how long it is or how many files it takes — see [deliverables.md](references/deliverables.md), which describes what `/audit:issues` produces without requiring the next skill to match it. - -## Shared references - -Skills are thin; the method lives once at plugin level. - -| Document | Owns | -|---|---| -| [references/doctrine.md](references/doctrine.md) | Trust rules, evidence classes, read-only boundary, checkpointing, coverage self-audit | -| [references/quantecon-context.md](references/quantecon-context.md) | Repo types, label ownership, the cross-repo graph, notes-system discovery, access | -| [references/deliverables.md](references/deliverables.md) | What an audit owes its reader, where reports may land, and the `/audit:issues` bundle as a worked example | -| [scripts/](scripts/) | Deterministic fetch machinery | - -## Running one - -``` -/audit:issues QuantEcon/action-translation -``` - -Audits work from a frozen snapshot and checkpoint to disk as they go, so an interrupted run resumes rather than restarting — and every number in the report refers to one point in time. - -**Budget tens of minutes, not hours.** The first measured run covered a 230-item tracker in **22 minutes** end to end ([record](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md)). What scales is the **open issue** count, not the item count: phase 2 verifies the open set at about 10 seconds each — 9 of those 22 minutes for 56 issues — while the remaining phases are largely fixed. So a 1000-item repo with a small open set is cheaper than a 300-item repo with a large one. That is one data point; a repo whose issues need deeper code archaeology will run slower per issue. - -Headless runs work the same way: - -```yaml -- uses: anthropics/claude-code-action@v1 - with: - plugin_marketplaces: "https://github.com/QuantEcon/skills.git" - plugins: "audit@quantecon" - prompt: "/audit:issues QuantEcon/action-translation" -``` - -## A note on naming - -Skill names here are objects because the plugin is the verb: `/audit:issues`, `/audit:translations`. Both read as commands, which is the part that matters, and `/audit:audit-issues` would stutter at every invocation. - -`audit` was chosen over `review` for the same reason the family excludes single-item work: `review` is already the per-item word here (`/benchmark:review-acceleration`, and PR review generally), so a `/review:prs` that sweeps every open PR would sit one keystroke from reviewing one. `audit` also matches QEP-3's `audit-` repo prefix and already connotes observe-and-report, which is the boundary this plugin enforces. diff --git a/benchmark/.claude-plugin/plugin.json b/benchmark/.claude-plugin/plugin.json deleted file mode 100644 index 142159f..0000000 --- a/benchmark/.claude-plugin/plugin.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "benchmark", - "description": "Benchmarking and acceleration-evaluation tools for QuantEcon lecture code", - "version": "0.4.0", - "author": { "name": "QuantEcon" } -} diff --git a/benchmark/CHANGELOG.md b/benchmark/CHANGELOG.md deleted file mode 100644 index 851fdc4..0000000 --- a/benchmark/CHANGELOG.md +++ /dev/null @@ -1,93 +0,0 @@ -# Changelog — `benchmark` - -Every released version of this plugin, newest first. A version exists because the content below shipped in it: the plugin cache is keyed by version string, so what you have installed is exactly the entries down to the version `claude plugin list` reports for `benchmark`. - -Versions are [semver](https://semver.org) as a user of this plugin experiences it — a new skill, or a procedure that now does something materially different, is a minor bump; a correction that leaves the procedure as it was is a patch. Nothing below 1.0.0 promises stability. - -Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/benchmark)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). - -## 0.4.0 — 2026-08-07 - -Triage becomes the front door, and every output leads with the decision. The reframing follows the maintainers' direction — the product most wanted is "look at a lecture and advise whether a JAX upgrade is recommended" — and the measured record agrees: in every evaluation to date (ge_arrow, markov_asset, wald_friedman, and the 2026-08-06 ge_arrow re-run on [skills#10](https://github.com/QuantEcon/skills/issues/10)) the recommendation was decided by the triage-layer instruments — the as-used baseline and what a conversion could reach — and never moved by the scorecard on top. Review mode is unchanged and stays: it is the mode that caught markov_asset's masked build defect, and it applies the day a conversion PR exists. - -**Changed** - -- `SKILL.md` leads with triage — the no-candidate "should this lecture be converted?" question — behind a "Which mode" router, with review as the second mode. The frontmatter description now opens with the advise use case, so natural-language invocation matches the common question. Review-mode content is unchanged. -- The scorer's printed output and the report format lead with the verdict. `score.py` prints `VERDICT:` above the weighted total, labels the total "for the record", and the deciding-flip lines name the verdict they flip to before the recomputed number (previously `⇒ total 2.30, …`, which two careful readers in a row took as the headline — [skills#14, finding 6](https://github.com/QuantEcon/skills/issues/14)). The report's TL;DR opens with the full verdict and carries the score alongside as candidate quality for the record; the dimension table gains a verdict row so it still carries the decision when quoted on its own. -- `README.md` puts triage first throughout — the mode table, the invocation examples, and the mode sections — and states that triage builds no candidate: it measures the lecture as it stands and bounds what a conversion could deliver. -- Triage now names its canonical decision criteria: the manual's JAX style page ([when to use JAX, when not to](https://manual.quantecon.org/styleguide/jax.html), including *Converting from Numba § Decide first*), cited rather than restated. The skill's four checks are framed as the measurement layer that tests whether those criteria hold for a given lecture — "a real bottleneck" is a claim the as-used baseline and pattern match establish or refute, while the page's "teaching JAX itself" criterion is editorial and stays a maintainer call. -- The triage decision rule no longer re-derives numbers from the rubric weights: it states the conclusion qualitatively and points to `references/EVALUATION_FRAMEWORK.md`, which the skill's own scoring step already said was the only place weights live. Triage's outcome vocabulary is standardized on **convert / don't-convert** in both `SKILL.md` and `README.md`. (Caught by Copilot's review of the 0.4.0 PR.) - -Nothing in the rubric, weights, gates, or scorecard JSON changed: the regression anchors (2.85 / 2.25) and the fixtures reproduce unchanged. - -## 0.3.2 — 2026-08-03 - -**Fixed** - -- The plugin README's status line said skill wiring was "tracked in skills#4". The wiring shipped in 0.3.0 — it is in that release's entry below — so the line pointed at an open issue for work that had already landed. 0.3.1 corrected the version number in that same sentence and left the stale clause standing, which is how a half-fixed line survives a review. It now describes the plugin as operational for workspace runs since 0.3.0 and points at this changelog for what shipped when. - -## 0.3.1 — 2026-08-03 - -**Added** - -- This changelog. - -**Fixed** - -- The plugin README's status line named `v0.2.0` — a version that was never released (see the note at the foot of this file). It now names `v0.3.0`, the release in which the evaluation system actually became runnable. That is a historical fact rather than a restatement of the current version, so it will not go stale again on the next bump. - -## 0.3.0 — 2026-07-27 - -The evaluation system became runnable: a deterministic scoring engine, rubric v2 with verdict gates, two complete worked evaluations to copy from, a triage mode, and the install fix that made the plugin installable at all. - -**Added** - -- A runnable scoring engine: `python scripts/scoring/score.py ` turns an evidence file into a scorecard. No score is ever typed by hand; the session shows the derivation table — every dimension score with the measured number and threshold band that produced it. -- `references/EVALUATION_FRAMEWORK.md` — the rubric in prose: seven weighted dimensions, numeric scoring anchors, structural checklists, verdict bands, worked HIGH/LOW examples. `SKILL.md` points here instead of restating weights, so recalibration cannot drift the copies. -- `scripts/scoring/EVIDENCE_TEMPLATE.json` — the judgement contract you fill in: measured numbers plus cited yes/no answers. -- Two complete worked evaluations in `references/examples/` (ge_arrow 2.85/5, markov_asset 2.25/5) with measurement scripts, results, evidence and reports — usable as per-lecture templates and as regression anchors, plus a README documenting where every evidence number came from. -- `scripts/calibration/bellman_bench.py` — the shared aiyagari Bellman benchmark that pins the "25x as-used = score 5" efficiency anchor. -- Rubric v2 verdict gates: the logic-and-design bug cap is derived from the correctness evidence (does it build, does it diverge under x64) rather than trusting a hand-set boolean, and the correctness score caps the verdict — a float32 catastrophe with no logic bug can no longer come out as "merge". -- A no-conversion verdict: a lecture whose baseline as-used total is under the 1 s materiality floor, with a slower candidate, now gets "don't convert" instead of a polished score of the rewrite. -- A sensitivity stamp on every scorecard: each scored input is perturbed one at a time (bools flipped, counts ±1, floats ±10%) and the verdict is stamped robust / fragile / robust-at-floor with the deciding flips listed. -- K-repeat as-used measurement: `run_all.py` repeats each side three times in fresh processes, the headline speedup is the median, and per-run spread feeds a contested-band annotation. -- Triage mode — "is this lecture worth converting at all?", answered from the existing lecture alone: baseline as-used total, workload-pattern match against the two calibrated poles, crossover check, readability-cost forecast, and the weight algebra that follows. Validated blind against the three known cases before being documented, including the documented limit that it cannot predict conversion-quality defects. -- `benchmark/README.md` — the plugin's user guide: review vs triage mode, the report format, the manual pipeline quickstart, and the one rule to remember (warm-only speedups are never the headline). -- Skill wiring for installed runs: evaluations are scaffolded under `/benchmark-eval//` with the plugin read-only at `${CLAUDE_PLUGIN_ROOT}`, preconditions stated up front, and an extraction/replay diff check so the replay provably matches the lecture. -- A provenance stamp written to `results/env.json` (python/platform/numpy/jax/quantecon versions), including the titles of any failed pipeline step so a partial run cannot claim full provenance. -- `references/fixtures/rubric_v2` — synthetic evidence whose only job is to execute five v2 code paths the worked examples never touch; every source string is prefixed `SYNTHETIC:` so the numbers cannot be cited as evidence about a lecture. - -**Changed** - -- The skill is now `/benchmark:review-acceleration`, renamed from `/benchmark:eval-py-acceleration`. The rename was authored on 2026-07-21 in [#1](https://github.com/QuantEcon/skills/pull/1) but reached installed users only with this version bump. -- `score.py` takes a lecture directory path and works from any working directory, instead of resolving a lecture name against a package root. -- Correction of record on markov_asset: the lecture does build in notebook order — a stale global `err` masks a stray `err.throw()`, silently disabling the checkify stability validation. Worse than a crash, but not the build failure the original report claimed; erratum prepended to the report and the wording fixed in the examples README, `SKILL.md` and the plugin README. -- Two earlier certifications withdrawn as overstated: the reference replays deviate from the lectures' construction patterns (not "mirrors the lecture exactly"), and the as-used totals were single-pass, not medians over repeats (v2 restores repeats explicitly). -- The plugin README's triage baselines are labelled as triage-time (2026-07-21) measurements, and the framework and `SKILL.md` stop restating them — the gate reads each lecture's own `baseline_as_used_seconds`. -- `SKILL.md` forbids reporting robust-at-floor as plain robust: a verdict already in the bottom band cannot be perturbed downward, so zero deciding flips there is band geometry, not evidence strength. - -**Fixed** - -- Install was broken for every user. The repo-level `.claude-plugin/marketplace.json` omitted the required top-level `owner`, and every plugin entry — this one included — used a remote source `{"source": "github", "repo": "QuantEcon/skills", "path": "benchmark"}` that forced an install-time SSH re-clone of this repo. All three entries switched to the co-located relative-path form (`"./benchmark"`), so install uses the marketplace copy already on disk: no SSH, no auth prerequisite. Surfaced by [@xuanguang-li](https://github.com/xuanguang-li) testing this plugin, [#10](https://github.com/QuantEcon/skills/issues/10). -- The verdict band is computed from the rounded total, so the band always agrees with the number shown — raw floating-point sums could land at 2.4999999999999996 for combinations that are exactly 2.50 (797 of 78125 score combinations affected). -- `matches_under_x64` now caps correctness on its own. The extra `max_delta_shipped > 1e-8` conjunct made the guard structurally unable to fire in exactly the "wrong economics masked by low precision" case it exists to catch — such a candidate scored correctness 5 / total 3.25; it now scores correctness 1 / total 2.30, gated to net regression. -- `score.py` validates evidence before scoring and refuses evidence that omits a scored input the gates read, or that marks a structural criterion met without a citation. A missing `baseline_as_used_seconds` silently disarmed the no-conversion verdict, and stripping every citation left the score unchanged. -- The headline metrics (as-used total, cold start) are persisted to `results/as_used.json` and `results/cold_start.json` with the derived speedup, instead of existing only on the console while the docstrings claimed aggregation. -- The sensitivity stamp's denominator is honest: perturbations that raise are recorded in `perturbations_skipped` rather than silently counted as tested. -- `run_all.py` hardened — JSON scalar stdout lines no longer abort the pipeline, per-step return codes are tracked, the as-used speedup derivation guards both sides, and duplicate mode keys warn instead of silently overwriting. -- ge_arrow's `check_equivalence.py` writes `equivalence_x64.json` under `JAX_ENABLE_X64` instead of clobbering the as-shipped results. -- ge_arrow static metrics double-counted concept-token hits via a duplicated pattern (informational metric; 110 → 105). -- markov_asset's `statements_for_one_asset` renamed to `statements_for_one_result` to match the evidence-template vocabulary (values unchanged). -- Two files that were CRLF (`references/EVALUATION_FRAMEWORK.md`, the ge_arrow report) are normalized to LF, so a future one-line edit no longer renders as a whole-file diff. - -## 0.1.0 — 2026-07-07 - -First release: the plugin appears in the marketplace with a documented but not yet runnable evaluation procedure — a v0 outline skill, no executable scripts. - -- `/benchmark:eval-py-acceleration` — a v0 outline of the acceleration-review procedure: the five steps (equivalence check, static metrics, as-used benchmark, seven-dimension scoring, report), the seven weights (readability 0.25 deliberately above efficiency 0.15), the verdict bands, and the two calibration anchors (aiyagari Bellman ~25x faster as-used = HIGH; ge_arrow ~45x slower as-used = LOW). -- The guiding principle a user is meant to apply: lectures are teaching materials first, so "uses JAX" is never a goal in itself. -- `scripts/README.md` listing the eight measurement scripts still to be collected from [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717). - ---- - -**There is no 0.2.0.** It existed on a branch inside [#5](https://github.com/QuantEcon/skills/pull/5) and was superseded within the same pull request; because the repo squash-merges, `main` went 0.1.0 → 0.3.0 in one commit and 0.2.0 was never published. Nothing is missing from this file. diff --git a/benchmark/scripts/README.md b/benchmark/scripts/README.md deleted file mode 100644 index defb300..0000000 --- a/benchmark/scripts/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# benchmark plugin — scripts - -The deterministic core of `/benchmark:review-acceleration`: the shared scoring engine and the efficiency calibration. Developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). - -## Layout - -``` -scoring/ - rubric.py the standard as code: evidence → score, deterministically - score.py engine/CLI: /evidence.json → results/scorecard.json - env_stamp.py provenance stamp: /results/env.json (+ failed steps) - EVIDENCE_TEMPLATE.json the judgement contract a new evaluation fills in -calibration/ - bellman_bench.py shared aiyagari Bellman benchmark — pins the "25× as-used - bellman_bench.json = score 5" efficiency anchor -``` - -All commands below run from the plugin root (`benchmark/` in this repo). - -The rubric in prose — dimensions, weights, anchors, checklists, verdict bands, worked HIGH/LOW examples — is [`../references/EVALUATION_FRAMEWORK.md`](../references/EVALUATION_FRAMEWORK.md). Two complete worked evaluations (measurement scripts, results, evidence, reports) live in [`../references/examples/`](../references/examples/) and double as the regression baseline the skill must reproduce. - -## How scoring works - -Scores are **never typed by hand** — each is a deterministic function of evidence: - -1. **Measure** — `python references/examples//scripts/run_all.py` runs the per-lecture measurement scripts and writes `results/*.json` plus a provenance stamp (`results/env.json`: Python/platform/library versions and any failed steps — the seed of the QuantEcon/meta#335 shared result schema; generated per-run, not committed). The as-used steps repeat 3× per side in fresh processes; the headline speedup is a **median**, with per-run values kept for the contested-band check. -2. **Record evidence** — fill `/evidence.json` (copy `scoring/EVIDENCE_TEMPLATE.json`): measured numbers into the quantitative slots with their source, and each structural checklist item answered true/false **with a citation to the diff**. -3. **Score** — `python scripts/scoring/score.py references/examples/` applies `rubric.py` and writes `results/scorecard.json`, printing the derivation of every score, the final verdict (after the v2 correctness gates and the no-conversion rule), and the one-flip **sensitivity stamp** (robust/fragile with deciding flips). - -## Evaluating a new lecture - -Per-lecture measurement scripts are **adapted templates, not a fixed harness** — copy an existing example and adapt (this is the step the skill automates): - -```bash -conda activate quantecon # jax 0.4.x, numpy 2.x, quantecon -mkdir -p references/examples//{scripts,results} -cp scripts/scoring/EVIDENCE_TEMPLATE.json references/examples//evidence.json -# drop in model_old.py (from main) and model_new.py (from the PR branch), -# adapt check_equivalence / static_metrics / benchmark / as_used_total from an -# existing example, wire them into run_all.py, then: -python references/examples//scripts/run_all.py -python scripts/scoring/score.py references/examples/ -# write _REPORT.md from the scorecard + evidence -``` - -Benchmarks are CPU-only; timings vary ±~15% run-to-run, so the rubric keys on orders of magnitude, not exact milliseconds. diff --git a/docs/developing-skills.md b/docs/developing-skills.md index 30ffefe..a406755 100644 --- a/docs/developing-skills.md +++ b/docs/developing-skills.md @@ -19,21 +19,21 @@ docs/ # these guides **A plugin needs its manifest and its changelog; a skill needs only `SKILL.md`.** A skill that is purely a procedure — nothing deterministic to run, no long reference material to point at — is one file in one directory, and should stay that way. `scripts/` appears when there is something mechanical worth doing in code; `references/` when the skill needs more context than belongs in its body. Adding either before you need it just makes the skill harder to read. -The three live plugins show some of the range: `qe` (standalone procedures — a PR-feedback loop with one script of its own, and a two-skill work-plan family: a lifecycle skill whose five verbs share one statement of the org's plan convention, plus a report-to-project packager), `benchmark` (one skill driving a deterministic engine, with worked examples as its regression baseline), and `audit` (sibling procedures sharing a method document). None of these is the house style — they are what three problems happened to need. +The live skill families show some of the range: the standalone procedures (a PR-feedback loop with one script of its own, and a two-skill work-plan family: a lifecycle skill whose five verbs share one statement of the org's plan convention, plus a report-to-project packager), `benchmark` (one skill driving a deterministic engine at `scripts/benchmark/`, with worked examples as its regression baseline), and the `audit-*` family (sibling procedures sharing a method document at `references/audit/`). None of these is the house style — they are what the problems happened to need. ## Conventions Guidance rather than gates. The repo is early, and most of what follows generalises from one or two worked examples; where something is genuinely load-bearing it says so and gives the reason. Departing from the rest is fine when you have a reason — and worth mentioning in the PR, since a second example is how any of this eventually becomes a real convention ([CATALOG.md § Principles](../CATALOG.md#principles)). -- **Invocations read as commands** — the whole `/plugin:skill` string, not the skill name alone. `/qe:check-style` (planned, [#3](https://github.com/QuantEcon/skills/issues/3)) puts the verb in the skill because `qe` names a domain; `/audit:issues` puts it in the plugin and leaves the skill as the object. Both read as imperatives, which is the only part that matters. There is no rule yet about which to prefer — three plugins is too few to know, so pick what reads best and let a convention emerge from use. +- **Invocations read as commands** — the whole `/qe:skill` string, not the skill name alone. Since the [#43](https://github.com/QuantEcon/skills/issues/43) consolidation there is one plugin, so the skill name carries the whole meaning: a verb where the skill acts (`/qe:check-style`, planned in [#3](https://github.com/QuantEcon/skills/issues/3)), a family stem plus object where a family may grow (`/qe:audit-issues`, keeping `audit-*` greppable). Either way the full string should read as an imperative. - **Description quality matters**: the SKILL.md frontmatter `description` is what natural-language invocation matches against. State what the skill does, what it measures, and when to use it. `validate.py` rejects descriptions too short to trigger reliably. - **Report first, fix on request** (load-bearing) — skills never silently edit; anything `build_risk` or output-changing (RNG streams) is presented, never auto-applied. - **Deterministic before LLM** (load-bearing, and the reason is that a reader has to be able to check a skill's output without re-running it): put what is mechanical in `scripts/` (checkable, testable, zero-false-positive bar); reserve the skill's judgement for what genuinely needs it. The discipline scales with what the skill outputs: 1. *Every skill*: claims carry citations — rule ID + `file:line`, or a number + its source. A findings list needs nothing more; don't add ceremony to simple skills. 2. *Skills that judge*: record judgement as discrete answers (true/false per criterion, each cited), not free prose — so it's checkable. - 3. *Skills that score*: when multiple judgements aggregate into a verdict with stakes, use the benchmark plugin's evidence-file pattern — judgement lives only in an evidence file, a deterministic engine computes every score, and no score is ever typed by hand. Aggregation is where hand-waving hides; the engine eliminates it. + 3. *Skills that score*: when multiple judgements aggregate into a verdict with stakes, use the benchmark skill's evidence-file pattern — judgement lives only in an evidence file, a deterministic engine computes every score, and no score is ever typed by hand. Aggregation is where hand-waving hides; the engine eliminates it. - **Don't duplicate content across docs** — one canonical location, pointers elsewhere. Rule text, weights, and thresholds especially: restated copies drift. -- **Self-contained plugins** (load-bearing — this one is a hard constraint of how plugins install, not a preference): an installed plugin ships only its own directory. No relative links or paths that escape the plugin root; use absolute GitHub URLs for repo-level files, and anchor runtime paths for the installed context with `${CLAUDE_PLUGIN_ROOT}` — `benchmark/skills/review-acceleration/SKILL.md` is the worked example. +- **Self-contained plugins** (load-bearing — this one is a hard constraint of how plugins install, not a preference): an installed plugin ships only its own directory. No relative links or paths that escape the plugin root; use absolute GitHub URLs for repo-level files, and anchor runtime paths for the installed context with `${CLAUDE_PLUGIN_ROOT}` — `qe/skills/benchmark/SKILL.md` is the worked example. ## Development loop @@ -51,7 +51,7 @@ Two tiers, fastest first. Either way, **test from a real consuming project** (a **Tier 1 — skill iteration, no install.** Load one plugin directly into a session: ```bash -claude --plugin-dir /path/to/skills/ # e.g. .../skills/benchmark +claude --plugin-dir /path/to/skills/qe ``` Nothing is installed and no marketplace state is touched. Best while editing SKILL.md or scripts; restart the session to pick up changes. @@ -60,7 +60,7 @@ Nothing is installed and no marketplace state is touched. Best while editing SKI ``` /plugin marketplace add /path/to/your/skills-checkout -/plugin install benchmark@quantecon +/plugin install qe@quantecon ``` Two things to know: @@ -73,7 +73,7 @@ Two things to know: ``` /plugin marketplace remove quantecon /plugin marketplace add QuantEcon/skills -/plugin install benchmark@quantecon +/plugin install qe@quantecon ``` Confirm with `/plugin marketplace list` (the source should read `QuantEcon/skills`, not your local path) and `/plugin list` (the version should match the merged `plugin.json`). Routine setup and updating for end users is covered in [using-skills.md](using-skills.md). @@ -87,7 +87,7 @@ npm install -g mystmd@1.10.1 # same version CI pins in docs.yml myst start # live-reloading preview ``` -Two things follow from how it is wired. **Pages are rendered from the files where they already live** — [`myst.yml`](../myst.yml) points at `benchmark/README.md` and `audit/references/*.md` in place, never a copy — so editing a plugin's docs updates the site, and no plugin loses documentation it needs to ship with it. And **the sources have to stay readable on GitHub**, since the repo view is the other half of the audience: prefer plain Markdown, and keep relative links relative, because CI checks that every one of them resolves. +Two things follow from how it is wired. **Pages are rendered from the files where they already live** — [`myst.yml`](../myst.yml) points at `qe/references/benchmark/README.md` and `qe/references/audit/*.md` in place, never a copy — so editing a plugin's docs updates the site, and no plugin loses documentation it needs to ship with it. And **the sources have to stay readable on GitHub**, since the repo view is the other half of the audience: prefer plain Markdown, and keep relative links relative, because CI checks that every one of them resolves. ## Versioning and releases @@ -95,8 +95,8 @@ The plugin is the released artifact and its version string is how a release is d - **A version bump is the delivery mechanism** (load-bearing — this is how the install cache works, not a convention we chose). An installed plugin lives at `~/.claude/plugins/cache////`, and `claude plugin update` compares version strings only: if the content changed and the version did not, it reports "already at the latest version" and refreshes nothing. Merged content with an unchanged version reaches nobody who has the plugin installed, and nothing warns you. This has already happened here three times — most recently [#27](https://github.com/QuantEcon/skills/pull/27), which edited a shipped `SKILL.md` under `qe/` without bumping `qe`. - **Therefore any change to any file under `/` bumps that plugin's version.** Every file in the directory ships; there is no non-shipping edit inside it, and an exception list is where a rule like this rots. A file *moved out* of a plugin counts too — what it ships changed either way. Repo-level files — `docs/`, `README.md`, `CATALOG.md`, CI, the marketplace manifest — ship to nobody and bump nothing. -- **Choose the number for what a user of the plugin experiences.** A new skill, or a procedure that now does something materially different: minor (benchmark's scaffolding → working evaluation system was 0.1.0 → 0.3.0 — there was never a published 0.2.0). A correction that leaves the procedure as it was: patch. Nothing below 1.0.0 promises stability. -- **The catalogue's own top-level `version` in `marketplace.json`** moves only when a plugin is added or removed. (Past practice is mixed — adding `qe` did not move it, adding `audit` did — so the rule is stated here rather than inferred.) It is not a cache key and delivers nothing, so it gets no changelog entry. +- **Choose the number for what a user of the plugin experiences.** A new skill, or a procedure that now does something materially different: minor (the retired benchmark stream's scaffolding → working evaluation system was 0.1.0 → 0.3.0 — there was never a published 0.2.0). A correction that leaves the procedure as it was: patch. Nothing below 1.0.0 promises stability. +- **The catalogue's own top-level `version` in `marketplace.json`** moves only when a plugin is added or removed. (Past practice is mixed — adding `qe` did not move it, adding `audit` did, and the #43 consolidation that removed two moved it again — so the rule is stated here rather than inferred.) It is not a cache key and delivers nothing, so it gets no changelog entry. ### The changelog diff --git a/docs/tutorial-run-an-audit.md b/docs/tutorial-run-an-audit.md index 577fab1..e9e06a7 100644 --- a/docs/tutorial-run-an-audit.md +++ b/docs/tutorial-run-an-audit.md @@ -1,10 +1,10 @@ # Tutorial: run a whole-tracker audit -This walks `/audit:issues` end to end against **[QuantEcon/action-translation](https://github.com/QuantEcon/action-translation)** — 228 items, the repo the runbook was first executed against by hand. +This walks `/qe:audit-issues` end to end against **[QuantEcon/action-translation](https://github.com/QuantEcon/action-translation)** — 228 items, the repo the runbook was first executed against by hand. -It differs from the [evaluation tutorial](tutorial-run-an-evaluation.md) in one important way. That one reproduces a committed reference, so every number you produce can be checked. Here there is no reference: `/audit:issues` has been run as a skill exactly **once** — run 1, against this same repo on 2026-07-28, which found seven plugin defects and is recorded [here](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md). A method generalised from one execution is still a hypothesis, so your run is the next data point in the validation program ([skills#16](https://github.com/QuantEcon/skills/issues/16)), and the part no automation can supply is your judgement of the output. Step 6 is therefore not optional garnish — it is the result. +It differs from the [evaluation tutorial](tutorial-run-an-evaluation.md) in one important way. That one reproduces a committed reference, so every number you produce can be checked. Here there is no reference: `/qe:audit-issues` has been run as a skill exactly **once** — run 1, against this same repo on 2026-07-28, which found seven plugin defects and is recorded [here](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md). A method generalised from one execution is still a hypothesis, so your run is the next data point in the validation program ([skills#16](https://github.com/QuantEcon/skills/issues/16)), and the part no automation can supply is your judgement of the output. Step 6 is therefore not optional garnish — it is the result. -Canonical references (this tutorial points, never restates): the procedure in [SKILL.md](../audit/skills/issues/SKILL.md), the method in [doctrine.md](../audit/references/doctrine.md), the org conventions in [quantecon-context.md](../audit/references/quantecon-context.md), the output contract in [deliverables.md](../audit/references/deliverables.md). +Canonical references (this tutorial points, never restates): the procedure in [SKILL.md](../qe/skills/audit-issues/SKILL.md), the method in [doctrine.md](../qe/references/audit/doctrine.md), the org conventions in [quantecon-context.md](../qe/references/audit/quantecon-context.md), the output contract in [deliverables.md](../qe/references/audit/deliverables.md). ## What you need @@ -17,14 +17,14 @@ Canonical references (this tutorial points, never restates): the procedure in [S ```bash claude plugin marketplace add QuantEcon/skills -claude plugin install audit@quantecon +claude plugin install qe@quantecon ``` **Then restart your session** — plugins register at startup, so the skill does not appear until you reopen. -The `/plugin marketplace add …` slash form does the same job, but it is a *terminal-CLI built-in*: the VS Code extension and the web app answer `/plugin isn't available in this environment`, while the `claude plugin` CLI above works from any shell. Confirm with `claude plugin list` — the version it reports should match the `audit` entry in [`marketplace.json`](../.claude-plugin/marketplace.json). (Naming a number here would go stale on the next release; if the two disagree, the install did not pick up the latest — `claude plugin update audit@quantecon`.) +The `/plugin marketplace add …` slash form does the same job, but it is a *terminal-CLI built-in*: the VS Code extension and the web app answer `/plugin isn't available in this environment`, while the `claude plugin` CLI above works from any shell. Confirm with `claude plugin list` — the version it reports should match the `qe` entry in [`marketplace.json`](../.claude-plugin/marketplace.json). (Naming a number here would go stale on the next release; if the two disagree, the install did not pick up the latest — `claude plugin update qe@quantecon`.) -If `/audit:issues` is still unrecognised after restarting, the plugin-prefixed slash form needs Claude Code 2.1.216+; the bare `/issues` works on older builds, and natural-language invocation ("audit every issue in this repo, output to …") works on any version ([using-skills § troubleshooting](using-skills.md#updating-and-troubleshooting)). +If `/qe:audit-issues` is still unrecognised after restarting, the plugin-prefixed slash form needs Claude Code 2.1.216+; the bare `/audit-issues` works on older builds, and natural-language invocation ("audit every issue in this repo, output to …") works on any version ([using-skills § troubleshooting](using-skills.md#updating-and-troubleshooting)). ## Step 1 — put the working directory where the repo already ignores it @@ -37,12 +37,12 @@ git check-ignore -v .dev/scratch/x # → .gitignore:… .dev/scratch/* **Be on the default branch, not merely clean.** Phase 2's core question is whether an issue still reproduces on `main`; run from a feature branch and every answer is measured against your unmerged work instead. A clean tree on the wrong branch passes the `git status` check and silently invalidates the phase the whole run exists to test — so check the branch, not just the status. -`action-translation` has a `.dev/` notes system whose `.dev/scratch/*` is already gitignored, which makes it the first-choice working directory: the run leaves `git status` completely clean, and no `.gitignore` edit is needed — that would itself be a change to a tracked file. Repos without one fall back to an untracked `.audit/` at the root ([SKILL.md § Working directory](../audit/skills/issues/SKILL.md)). +`action-translation` has a `.dev/` notes system whose `.dev/scratch/*` is already gitignored, which makes it the first-choice working directory: the run leaves `git status` completely clean, and no `.gitignore` edit is needed — that would itself be a change to a tracked file. Repos without one fall back to an untracked `.audit/` at the root ([SKILL.md § Working directory](../qe/skills/audit-issues/SKILL.md)). ## Step 2 — invoke ``` -/audit:issues QuantEcon/action-translation --out .dev/scratch/audit-2026-07-28 +/qe:audit-issues QuantEcon/action-translation --out .dev/scratch/audit-2026-07-28 ``` Before phase 1 the skill *discovers* its inputs rather than asking for them: the notes system (here `.dev/` — `STATE.md`, `PLAN.md`, `FUTURE.md`, `decisions/`), the label policy, the work-plan anchor to tier against, and any prior audits. It should ask you only where discovery is genuinely ambiguous — two plausible plan anchors, say — and never merely because something came up empty. **Every resolved input must appear in the report's method section**; that is the first thing to check in Step 6. @@ -76,7 +76,7 @@ The long phase: 116 items checked against the default branch rather than against **Interrupt during the closed pass too, if you get the chance** — that is the half that was not checkpointed at all before #34, so nothing has ever resumed from it. Resumability is asserted in three separate files and has never been tested. Two fixes have gone in ahead of this run and neither has been exercised: [#17](https://github.com/QuantEcon/skills/pull/17) named the artifacts, since before it phases 2 and 3 named none and a resumed session could only work by inventing the same filename; and [#34](https://github.com/QuantEcon/skills/pull/34) made the checkpoint cover both passes, since run 1 wrote only the open set and sent 62 closed issues straight to the catalog. This run is what checks both. -While it runs, `tail findings.md` occasionally. Every status claim should carry `[verified]`, `[stated]` or `[inferred]`, and a `[verified]` should cite `file:line`, a merged PR, a tag, or a commit — **and whatever it cites must resolve on the ref the audit named**, never a comment. A citation that only resolves in the author's working tree or on an unmerged branch is the defect [doctrine §2](../audit/references/doctrine.md#2-evidence-classes) now rules out; run 1's headline finding had exactly that shape. +While it runs, `tail findings.md` occasionally. Every status claim should carry `[verified]`, `[stated]` or `[inferred]`, and a `[verified]` should cite `file:line`, a merged PR, a tag, or a commit — **and whatever it cites must resolve on the ref the audit named**, never a comment. A citation that only resolves in the author's working tree or on an unmerged branch is the defect [doctrine §2](../qe/references/audit/doctrine.md#2-evidence-classes) now rules out; run 1's headline finding had exactly that shape. ## Step 5 — phases 3 to 5 diff --git a/docs/tutorial-run-an-evaluation.md b/docs/tutorial-run-an-evaluation.md index 75a9fd9..65449db 100644 --- a/docs/tutorial-run-an-evaluation.md +++ b/docs/tutorial-run-an-evaluation.md @@ -1,12 +1,12 @@ # Tutorial: run a full evaluation by hand -This walks the `/benchmark:review-acceleration` procedure end-to-end **by hand**, using the recorded [ge_arrow validation run](../reviews/validation-run-ge_arrow-2026-07-22.md) as the worked example — so every number you produce can be checked against a committed reference. When you invoke the skill, Claude drives these same steps for you; doing it manually once is the fastest way to understand what the skill measures, what the scorecard means, and how to debug a run that goes wrong. +This walks the `/qe:benchmark` procedure end-to-end **by hand**, using the recorded [ge_arrow validation run](../reviews/validation-run-ge_arrow-2026-07-22.md) as the worked example — so every number you produce can be checked against a committed reference. When you invoke the skill, Claude drives these same steps for you; doing it manually once is the fastest way to understand what the skill measures, what the scorecard means, and how to debug a run that goes wrong. -Canonical references (this tutorial points, never restates): the procedure in [SKILL.md](../benchmark/skills/review-acceleration/SKILL.md), the rubric in [EVALUATION_FRAMEWORK.md](../benchmark/references/EVALUATION_FRAMEWORK.md), the engine contract in [scripts/README.md](../benchmark/scripts/README.md). +Canonical references (this tutorial points, never restates): the procedure in [SKILL.md](../qe/skills/benchmark/SKILL.md), the rubric in [EVALUATION_FRAMEWORK.md](../qe/references/benchmark/EVALUATION_FRAMEWORK.md), the engine contract in [scripts/README.md](../qe/scripts/benchmark/README.md). ## What you need -- The `benchmark` plugin installed (or this repo checked out — its `benchmark/` directory serves as the plugin root). +- The `qe` plugin installed (or this repo checked out — its `qe/` directory serves as the plugin root). - A Python environment with `jax`, `numpy`, and the lecture's imports. The reference used jax 0.4.35; the validation run used jax **0.10.1** — the verdict reproduced anyway, which is the point of band-based scoring. - A checkout of the lecture repo. Evaluations always compare two refs: a **baseline** (the lecture before the conversion) and a **candidate** (the conversion PR's head). @@ -28,10 +28,10 @@ The merge-base is the "world before the PR" — that's the baseline. The PR bran Evaluations live in **your workspace**, never inside the plugin (which is read-only when installed): ```bash -export CLAUDE_PLUGIN_ROOT=/path/to/plugin/benchmark # or the installed plugin root +export CLAUDE_PLUGIN_ROOT=/path/to/skills/qe # or the installed plugin root mkdir -p benchmark-eval/ge_arrow/scripts -cp $CLAUDE_PLUGIN_ROOT/references/examples/ge_arrow/scripts/*.py benchmark-eval/ge_arrow/scripts/ -cp $CLAUDE_PLUGIN_ROOT/scripts/scoring/EVIDENCE_TEMPLATE.json benchmark-eval/ge_arrow/evidence.json +cp $CLAUDE_PLUGIN_ROOT/references/benchmark/examples/ge_arrow/scripts/*.py benchmark-eval/ge_arrow/scripts/ +cp $CLAUDE_PLUGIN_ROOT/scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json benchmark-eval/ge_arrow/evidence.json ``` Because we are *reproducing* the ge_arrow evaluation, we copy its already-adapted scripts. For a **new** lecture you adapt them — extract `model_old.py` from the lecture at the baseline ref and `model_new.py` at the candidate ref **verbatim** (disclose any deviation), and rewrite the measurement scripts around the lecture's actual examples and call sequence. That adaptation is the skill's real work; there is deliberately no rigid harness. Either way, before measuring, diff your extractions against the lecture's cells — the validation run did exactly this and caught an undisclosed whitespace normalisation in the committed baseline extraction. @@ -74,7 +74,7 @@ Fill `benchmark-eval/ge_arrow/evidence.json` from `results/`: measured numbers i ## Step 4 — score ```bash -python $CLAUDE_PLUGIN_ROOT/scripts/scoring/score.py benchmark-eval/ge_arrow +python $CLAUDE_PLUGIN_ROOT/scripts/benchmark/scoring/score.py benchmark-eval/ge_arrow ``` No score is ever typed by hand — the engine computes all seven dimensions and prints the derivation of each. The validation run's tail: @@ -110,4 +110,4 @@ If your bands move, something real changed — check `results/env.json` first, t ## Step 6 — report -Write `_REPORT.md` from the scorecard + evidence following the worked examples' format ([ge_arrow](../benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md), [markov_asset](../benchmark/references/examples/markov_asset/markov_asset_REPORT.md)): TL;DR with the full verdict, dimension table, evidence per dimension, and a must-fix list. For the validation run the "report" is the [cross-comparison record](../reviews/validation-run-ge_arrow-2026-07-22.md) itself. +Write `_REPORT.md` from the scorecard + evidence following the worked examples' format ([ge_arrow](../qe/references/benchmark/examples/ge_arrow/ge_arrow_REPORT.md), [markov_asset](../qe/references/benchmark/examples/markov_asset/markov_asset_REPORT.md)): TL;DR with the full verdict, dimension table, evidence per dimension, and a must-fix list. For the validation run the "report" is the [cross-comparison record](../reviews/validation-run-ge_arrow-2026-07-22.md) itself. diff --git a/docs/using-skills.md b/docs/using-skills.md index 15b7207..234edf1 100644 --- a/docs/using-skills.md +++ b/docs/using-skills.md @@ -6,20 +6,18 @@ For lecture authors, reviewers, and maintainers who want to *use* the skills. (B **In a lecture repo that has opted in** — nothing to do. Repos that check the marketplace into `.claude/settings.json` (see the [repo README](../README.md)) install everything automatically when you open the repo in Claude Code and trust it. -**Anywhere else** — three commands in a Claude Code session (the marketplace first, then the plugins you want): +**Anywhere else** — two commands in a Claude Code session (the marketplace, then the one plugin): ``` /plugin marketplace add QuantEcon/skills -/plugin install qe@quantecon # author-facing base skills -/plugin install benchmark@quantecon # lecture-evaluation tooling -/plugin install audit@quantecon # maintainer-facing bulk audits +/plugin install qe@quantecon ``` **`/plugin` is a terminal-CLI built-in**, so the VS Code extension and the web app answer `/plugin isn't available in this environment`. The `claude plugin` CLI does the same job from any shell and works everywhere: ```bash claude plugin marketplace add QuantEcon/skills -claude plugin install audit@quantecon # and/or qe@quantecon, benchmark@quantecon +claude plugin install qe@quantecon ``` Either route, **restart the session afterwards** — plugins register at startup, so a newly installed skill is absent until you reopen. `claude plugin list` shows what is installed and at which version. @@ -30,29 +28,29 @@ Either route, **restart the session afterwards** — plugins register at startup Two ways, equivalent: -1. **Slash command** — type `/` and pick from the menu, e.g. `/benchmark:review-acceleration 717`. Trailing words are passed to the skill as arguments. +1. **Slash command** — type `/` and pick from the menu, e.g. `/qe:benchmark 717`. Trailing words are passed to the skill as arguments. 2. **Natural language** — describe what you want ("work through Copilot's review of this PR"; "is this JAX conversion actually an improvement?") and the matching skill triggers on its description. ## What to expect - **Report first, fix on request.** Skills produce a structured report and *offer* changes; they never silently edit your files. Risky fixes (anything that breaks builds or changes published figures, e.g. RNG-stream changes) are presented but never auto-applied. -- **Evidence, not vibes.** Reports cite rule IDs, `file:line` locations, and measured numbers. The benchmark plugin goes further: its scores are computed by a deterministic engine from recorded evidence — the session shows the full derivation. +- **Evidence, not vibes.** Reports cite rule IDs, `file:line` locations, and measured numbers. The benchmark skill goes further: its scores are computed by a deterministic engine from recorded evidence — the session shows the full derivation. - **The same skill works pre-PR and in review.** Run it on your working copy before opening a PR (catch issues early), or point it at an open PR (consistent review). ## The plugins -Everything that registers in your slash menu. Since `qe` 0.6.0 every entry is a skill that actually runs — an unbuilt skill lives only as the plan in its plugin's tracking issue (the style-check family, for example, is [skills#3](https://github.com/QuantEcon/skills/issues/3)). [CATALOG.md](../CATALOG.md) is the stricter list: merged, operational, *and* stating how far each has been validated. +Everything that registers in your slash menu — one plugin, five skills. Since `qe` 0.6.0 every entry is a skill that actually runs — an unbuilt skill lives only as the plan in its family's tracking issue (the style-check family, for example, is [skills#3](https://github.com/QuantEcon/skills/issues/3)). [CATALOG.md](../CATALOG.md) is the stricter list: merged, operational, *and* stating how far each has been validated. -| Plugin | Skills | What they do | Status | -|---|---|---|---| -| `qe` | `/qe:copilot-review` | Work through GitHub Copilot's review of a PR: a verdict and recommended fix per comment, then a threaded reply to each one so they can be resolved from the GitHub UI | operational, validated from an installed plugin 2026-08-03 — [#26](https://github.com/QuantEcon/skills/pull/26) | -| `qe` | `/qe:workplan-project`, `/qe:workplan` | The work-plan family: `workplan-project` turns an audit/review report into a tracking issue with linked sub-issues; `workplan` carries the single work-plan issue that holds state between agent sessions through its lifecycle — `create`, `read` (validate against live state and recommend next steps; writes nothing), `resume`, `update`, `close`-and-succeed. All GitHub writes are drafted first and gated on your approval | merged as complete procedures, no validated run yet — [skills#3](https://github.com/QuantEcon/skills/issues/3) | -| `benchmark` | `/benchmark:review-acceleration` | Advise whether a lecture is worth converting at all (triage — the front door), or score a submitted NumPy→JAX/Numba conversion against the rubric (review) | operational for workspace runs — [guide](../benchmark/README.md), [skills#4](https://github.com/QuantEcon/skills/issues/4) | -| `audit` | `/audit:issues` | Sweep a whole tracker: verify each issue's status against the code rather than the thread, tier the open set into the repo's plan, deliver a report bundle. Read-only — it recommends, never applies | run once as a skill, method validated only that far — [tutorial](tutorial-run-an-audit.md), [guide](../audit/README.md), [skills#16](https://github.com/QuantEcon/skills/issues/16) | +| Skill | What it does | Status | +|---|---|---| +| `/qe:copilot-review` | Work through GitHub Copilot's review of a PR: a verdict and recommended fix per comment, then a threaded reply to each one so they can be resolved from the GitHub UI | operational, validated from an installed plugin 2026-08-03 — [#26](https://github.com/QuantEcon/skills/pull/26) | +| `/qe:workplan-project`, `/qe:workplan` | The work-plan family: `workplan-project` turns an audit/review report into a tracking issue with linked sub-issues; `workplan` carries the single work-plan issue that holds state between agent sessions through its lifecycle — `create`, `read` (validate against live state and recommend next steps; writes nothing), `resume`, `update`, `close`-and-succeed. All GitHub writes are drafted first and gated on your approval | merged as complete procedures, no validated run yet — [skills#3](https://github.com/QuantEcon/skills/issues/3) | +| `/qe:benchmark` | Advise whether a lecture is worth converting at all (triage — the front door), or score a submitted NumPy→JAX/Numba conversion against the rubric (review) | operational for workspace runs — [guide](../qe/references/benchmark/README.md), [skills#4](https://github.com/QuantEcon/skills/issues/4) | +| `/qe:audit-issues` | Sweep a whole tracker: verify each issue's status against the code rather than the thread, tier the open set into the repo's plan, deliver a report bundle. Read-only — it recommends, never applies | run once as a skill, method validated only that far — [tutorial](tutorial-run-an-audit.md), [guide](../qe/references/audit/README.md), [skills#16](https://github.com/QuantEcon/skills/issues/16) | ## Updating and troubleshooting -- **Update**: `claude plugin update @quantecon` (per plugin, from any shell — e.g. `qe@quantecon`), or `/plugin` → marketplace → update; repos with the settings.json opt-in track the marketplace automatically. Like installs, updates apply on the next session restart. +- **Update**: `claude plugin update qe@quantecon` from any shell, or `/plugin` → marketplace → update; repos with the settings.json opt-in track the marketplace automatically. Like installs, updates apply on the next session restart. - **Skill not in the menu?** Check the plugin is installed and enabled (`/plugin`), and that you trusted the repo. In settings-managed repos, `enabledPlugins` must list it. -- **`Unknown command: /benchmark:review-acceleration`?** The plugin-prefixed slash form needs a recent Claude Code (v2.1.216+; check with `claude --version`). On older versions the skill still registers under the bare `/review-acceleration`, and **natural-language invocation works on any version** — just describe the task ("is this JAX conversion worth merging?"). If it resolves under none of these, the install didn't complete — re-run `/plugin install benchmark@quantecon`. +- **`Unknown command: /qe:benchmark`?** The plugin-prefixed slash form needs a recent Claude Code (v2.1.216+; check with `claude --version`). On older versions the skill still registers under the bare `/benchmark`, and **natural-language invocation works on any version** — just describe the task ("is this JAX conversion worth merging?"). If it resolves under none of these, the install didn't complete — re-run `/plugin install qe@quantecon`. And if the *old* names (`/benchmark:review-acceleration`, `/audit:issues`) still appear, the retired plugins are lingering — `claude plugin uninstall benchmark@quantecon audit@quantecon`. - **Version pinning**: plugin versions live in the marketplace catalogue; CI validates that every manifest is consistent, so a broken install is a bug — please open an issue. diff --git a/myst.yml b/myst.yml index c15cb55..92e5f65 100644 --- a/myst.yml +++ b/myst.yml @@ -23,18 +23,18 @@ project: - file: docs/developing-skills.md - file: docs/tutorial-run-an-evaluation.md - file: docs/tutorial-run-an-audit.md - - title: Plugins + - title: Skill families children: - - file: benchmark/README.md - title: Benchmark plugin + - file: qe/references/benchmark/README.md + title: The benchmark skill children: - - file: benchmark/references/EVALUATION_FRAMEWORK.md - - file: audit/README.md - title: Audit plugin + - file: qe/references/benchmark/EVALUATION_FRAMEWORK.md + - file: qe/references/audit/README.md + title: The audit skills children: - - file: audit/references/doctrine.md - - file: audit/references/quantecon-context.md - - file: audit/references/deliverables.md + - file: qe/references/audit/doctrine.md + - file: qe/references/audit/quantecon-context.md + - file: qe/references/audit/deliverables.md - title: Project children: - file: CATALOG.md diff --git a/qe/.claude-plugin/plugin.json b/qe/.claude-plugin/plugin.json index bf62506..f7eeb26 100644 --- a/qe/.claude-plugin/plugin.json +++ b/qe/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "qe", - "description": "QuantEcon's author-facing base skills — working through a PR's review feedback, turning audit reports into tracked work projects, and creating, reading and carrying work-plan state across agent sessions", - "version": "0.6.0", + "description": "QuantEcon's shared agent skills under one namespace — working through a PR's review feedback, advising on and scoring lecture acceleration (benchmark), bulk read-only repository audits, turning audit reports into tracked work projects, and creating, reading and carrying work-plan state across agent sessions", + "version": "0.7.0", "author": { "name": "QuantEcon" } } diff --git a/qe/CHANGELOG.md b/qe/CHANGELOG.md index 6165c8c..72992ea 100644 --- a/qe/CHANGELOG.md +++ b/qe/CHANGELOG.md @@ -6,6 +6,24 @@ Versions are [semver](https://semver.org) as a user of this plugin experiences i Repository: [QuantEcon/skills](https://github.com/QuantEcon/skills) ([every commit that touched this plugin](https://github.com/QuantEcon/skills/commits/main/qe)). How a release is made: [developing-skills § Versioning and releases](https://github.com/QuantEcon/skills/blob/main/docs/developing-skills.md#versioning-and-releases). +## 0.7.0 — 2026-08-25 + +One namespace ([#43](https://github.com/QuantEcon/skills/issues/43)): the `benchmark` and `audit` plugins fold into `qe`, so every invocation reads as a QuantEcon skill and the catalog is one flat, small list. Three plugin prefixes encoded an installation distinction users don't care about when typing a command. This release starts strictly above every retiring stream (qe 0.6.0, benchmark 0.4.0, audit 0.2.0), so no version number in this merged changelog ever names two trees; the retired plugins' own entries are preserved below as historical sections. + +**Added** + +- `/qe:benchmark` — the benchmark plugin's `/benchmark:review-acceleration`, renamed to match how everyone refers to it; triage vs review stays mode selection from the arguments. Procedure unchanged; the scoring engine now lives at `scripts/benchmark/scoring/` and the framework, worked examples and fixtures at `references/benchmark/`, with `${CLAUDE_PLUGIN_ROOT}` paths updated throughout (including the worked examples' `run_all.py` engine lookup and its repo-layout fallback). +- `/qe:audit-issues` — the audit plugin's `/audit:issues`; the `audit-` stem keeps the family greppable if `/qe:audit-prs` or `/qe:audit-translations` ever pass their validation gate. Procedure unchanged; the shared method docs live at `references/audit/` and the snapshot fetcher at `scripts/audit/`. + +**Removed** + +- The separate `benchmark` and `audit` plugins. **Migration for installed users**: run `claude plugin uninstall benchmark@quantecon audit@quantecon` (or the `/plugin` menu equivalent), or the retired skills linger in the slash menu under their old names; lecture repos drop `benchmark@quantecon` from `enabledPlugins` in `.claude/settings.json`. One consequence is deliberate and worth knowing: the plugin is the enable unit, so every `qe` consumer now gets the audit skills too — two extra read-only entries in a five-item menu, judged an acceptable trade in #43. + +**Changed** + +- The plugin description now names the whole surface: PR review feedback, acceleration triage/review, bulk audits, report-to-project, and the work-plan lifecycle. +- Old tags (`benchmark--v0.4.0`, `audit--v0.2.0`, …) remain valid archaeology for the retired streams. + ## 0.6.0 — 2026-08-25 The style-check scaffolding leaves the shipped plugin. The seven `check-*` skills had reported "not yet operational" since they merged in [#2](https://github.com/QuantEcon/skills/pull/2), because everything that would make them work — the rendered rules snapshot, the deterministic preflight — is still pending upstream (see [issue #3](https://github.com/QuantEcon/skills/issues/3)). Shipping menu entries that do nothing costs more than it signals: they occupy seven slots in every consumer's slash menu and every description competes for natural-language routing against skills that actually run. The plan is unchanged and stays in issue #3, where unbuilt work belongs; the skills return operational, not as scaffolding. @@ -134,3 +152,182 @@ First release. The author-facing style-check surface appears in the slash menu a --- **Before this file existed**, two changes to `qe/` shipped without a version bump, so two different trees have been distributed under one version string each. Under 0.1.0, [#5](https://github.com/QuantEcon/skills/pull/5) rewrote the status banner in all seven `check-*` skills to point at [issue #3](https://github.com/QuantEcon/skills/issues/3) instead of `CATALOG.md`. Under 0.2.0, [#27](https://github.com/QuantEcon/skills/pull/27) made the correction now released as 0.2.1. If your install predates those dates, `claude plugin update` will not have reconciled it — reinstalling at 0.2.1 gets you the current tree. The [CI guard](https://github.com/QuantEcon/skills/blob/main/scripts/check-version-bump.py) landed alongside this release is what stops it happening again. + +--- + +## Historical — the `benchmark` plugin (folded into `qe` at 0.7.0) + +Released 2026-07-07 to 2026-08-07 as the separate `benchmark` plugin; its skill lives on as `/qe:benchmark`. These entries are frozen as shipped — version numbers and tags (`benchmark--v0.4.0`) name the retired plugin's own stream, which qe's 0.7.0 starts strictly above so no number ever names two trees. + +### benchmark 0.4.0 — 2026-08-07 + +Triage becomes the front door, and every output leads with the decision. The reframing follows the maintainers' direction — the product most wanted is "look at a lecture and advise whether a JAX upgrade is recommended" — and the measured record agrees: in every evaluation to date (ge_arrow, markov_asset, wald_friedman, and the 2026-08-06 ge_arrow re-run on [skills#10](https://github.com/QuantEcon/skills/issues/10)) the recommendation was decided by the triage-layer instruments — the as-used baseline and what a conversion could reach — and never moved by the scorecard on top. Review mode is unchanged and stays: it is the mode that caught markov_asset's masked build defect, and it applies the day a conversion PR exists. + +**Changed** + +- `SKILL.md` leads with triage — the no-candidate "should this lecture be converted?" question — behind a "Which mode" router, with review as the second mode. The frontmatter description now opens with the advise use case, so natural-language invocation matches the common question. Review-mode content is unchanged. +- The scorer's printed output and the report format lead with the verdict. `score.py` prints `VERDICT:` above the weighted total, labels the total "for the record", and the deciding-flip lines name the verdict they flip to before the recomputed number (previously `⇒ total 2.30, …`, which two careful readers in a row took as the headline — [skills#14, finding 6](https://github.com/QuantEcon/skills/issues/14)). The report's TL;DR opens with the full verdict and carries the score alongside as candidate quality for the record; the dimension table gains a verdict row so it still carries the decision when quoted on its own. +- `README.md` puts triage first throughout — the mode table, the invocation examples, and the mode sections — and states that triage builds no candidate: it measures the lecture as it stands and bounds what a conversion could deliver. +- Triage now names its canonical decision criteria: the manual's JAX style page ([when to use JAX, when not to](https://manual.quantecon.org/styleguide/jax.html), including *Converting from Numba § Decide first*), cited rather than restated. The skill's four checks are framed as the measurement layer that tests whether those criteria hold for a given lecture — "a real bottleneck" is a claim the as-used baseline and pattern match establish or refute, while the page's "teaching JAX itself" criterion is editorial and stays a maintainer call. +- The triage decision rule no longer re-derives numbers from the rubric weights: it states the conclusion qualitatively and points to `references/EVALUATION_FRAMEWORK.md`, which the skill's own scoring step already said was the only place weights live. Triage's outcome vocabulary is standardized on **convert / don't-convert** in both `SKILL.md` and `README.md`. (Caught by Copilot's review of the 0.4.0 PR.) + +Nothing in the rubric, weights, gates, or scorecard JSON changed: the regression anchors (2.85 / 2.25) and the fixtures reproduce unchanged. + +### benchmark 0.3.2 — 2026-08-03 + +**Fixed** + +- The plugin README's status line said skill wiring was "tracked in skills#4". The wiring shipped in 0.3.0 — it is in that release's entry below — so the line pointed at an open issue for work that had already landed. 0.3.1 corrected the version number in that same sentence and left the stale clause standing, which is how a half-fixed line survives a review. It now describes the plugin as operational for workspace runs since 0.3.0 and points at this changelog for what shipped when. + +### benchmark 0.3.1 — 2026-08-03 + +**Added** + +- This changelog. + +**Fixed** + +- The plugin README's status line named `v0.2.0` — a version that was never released (see the note at the foot of this file). It now names `v0.3.0`, the release in which the evaluation system actually became runnable. That is a historical fact rather than a restatement of the current version, so it will not go stale again on the next bump. + +### benchmark 0.3.0 — 2026-07-27 + +The evaluation system became runnable: a deterministic scoring engine, rubric v2 with verdict gates, two complete worked evaluations to copy from, a triage mode, and the install fix that made the plugin installable at all. + +**Added** + +- A runnable scoring engine: `python scripts/scoring/score.py ` turns an evidence file into a scorecard. No score is ever typed by hand; the session shows the derivation table — every dimension score with the measured number and threshold band that produced it. +- `references/EVALUATION_FRAMEWORK.md` — the rubric in prose: seven weighted dimensions, numeric scoring anchors, structural checklists, verdict bands, worked HIGH/LOW examples. `SKILL.md` points here instead of restating weights, so recalibration cannot drift the copies. +- `scripts/scoring/EVIDENCE_TEMPLATE.json` — the judgement contract you fill in: measured numbers plus cited yes/no answers. +- Two complete worked evaluations in `references/examples/` (ge_arrow 2.85/5, markov_asset 2.25/5) with measurement scripts, results, evidence and reports — usable as per-lecture templates and as regression anchors, plus a README documenting where every evidence number came from. +- `scripts/calibration/bellman_bench.py` — the shared aiyagari Bellman benchmark that pins the "25x as-used = score 5" efficiency anchor. +- Rubric v2 verdict gates: the logic-and-design bug cap is derived from the correctness evidence (does it build, does it diverge under x64) rather than trusting a hand-set boolean, and the correctness score caps the verdict — a float32 catastrophe with no logic bug can no longer come out as "merge". +- A no-conversion verdict: a lecture whose baseline as-used total is under the 1 s materiality floor, with a slower candidate, now gets "don't convert" instead of a polished score of the rewrite. +- A sensitivity stamp on every scorecard: each scored input is perturbed one at a time (bools flipped, counts ±1, floats ±10%) and the verdict is stamped robust / fragile / robust-at-floor with the deciding flips listed. +- K-repeat as-used measurement: `run_all.py` repeats each side three times in fresh processes, the headline speedup is the median, and per-run spread feeds a contested-band annotation. +- Triage mode — "is this lecture worth converting at all?", answered from the existing lecture alone: baseline as-used total, workload-pattern match against the two calibrated poles, crossover check, readability-cost forecast, and the weight algebra that follows. Validated blind against the three known cases before being documented, including the documented limit that it cannot predict conversion-quality defects. +- `benchmark/README.md` — the plugin's user guide: review vs triage mode, the report format, the manual pipeline quickstart, and the one rule to remember (warm-only speedups are never the headline). +- Skill wiring for installed runs: evaluations are scaffolded under `/benchmark-eval//` with the plugin read-only at `${CLAUDE_PLUGIN_ROOT}`, preconditions stated up front, and an extraction/replay diff check so the replay provably matches the lecture. +- A provenance stamp written to `results/env.json` (python/platform/numpy/jax/quantecon versions), including the titles of any failed pipeline step so a partial run cannot claim full provenance. +- `references/fixtures/rubric_v2` — synthetic evidence whose only job is to execute five v2 code paths the worked examples never touch; every source string is prefixed `SYNTHETIC:` so the numbers cannot be cited as evidence about a lecture. + +**Changed** + +- The skill is now `/benchmark:review-acceleration`, renamed from `/benchmark:eval-py-acceleration`. The rename was authored on 2026-07-21 in [#1](https://github.com/QuantEcon/skills/pull/1) but reached installed users only with this version bump. +- `score.py` takes a lecture directory path and works from any working directory, instead of resolving a lecture name against a package root. +- Correction of record on markov_asset: the lecture does build in notebook order — a stale global `err` masks a stray `err.throw()`, silently disabling the checkify stability validation. Worse than a crash, but not the build failure the original report claimed; erratum prepended to the report and the wording fixed in the examples README, `SKILL.md` and the plugin README. +- Two earlier certifications withdrawn as overstated: the reference replays deviate from the lectures' construction patterns (not "mirrors the lecture exactly"), and the as-used totals were single-pass, not medians over repeats (v2 restores repeats explicitly). +- The plugin README's triage baselines are labelled as triage-time (2026-07-21) measurements, and the framework and `SKILL.md` stop restating them — the gate reads each lecture's own `baseline_as_used_seconds`. +- `SKILL.md` forbids reporting robust-at-floor as plain robust: a verdict already in the bottom band cannot be perturbed downward, so zero deciding flips there is band geometry, not evidence strength. + +**Fixed** + +- Install was broken for every user. The repo-level `.claude-plugin/marketplace.json` omitted the required top-level `owner`, and every plugin entry — this one included — used a remote source `{"source": "github", "repo": "QuantEcon/skills", "path": "benchmark"}` that forced an install-time SSH re-clone of this repo. All three entries switched to the co-located relative-path form (`"./benchmark"`), so install uses the marketplace copy already on disk: no SSH, no auth prerequisite. Surfaced by [@xuanguang-li](https://github.com/xuanguang-li) testing this plugin, [#10](https://github.com/QuantEcon/skills/issues/10). +- The verdict band is computed from the rounded total, so the band always agrees with the number shown — raw floating-point sums could land at 2.4999999999999996 for combinations that are exactly 2.50 (797 of 78125 score combinations affected). +- `matches_under_x64` now caps correctness on its own. The extra `max_delta_shipped > 1e-8` conjunct made the guard structurally unable to fire in exactly the "wrong economics masked by low precision" case it exists to catch — such a candidate scored correctness 5 / total 3.25; it now scores correctness 1 / total 2.30, gated to net regression. +- `score.py` validates evidence before scoring and refuses evidence that omits a scored input the gates read, or that marks a structural criterion met without a citation. A missing `baseline_as_used_seconds` silently disarmed the no-conversion verdict, and stripping every citation left the score unchanged. +- The headline metrics (as-used total, cold start) are persisted to `results/as_used.json` and `results/cold_start.json` with the derived speedup, instead of existing only on the console while the docstrings claimed aggregation. +- The sensitivity stamp's denominator is honest: perturbations that raise are recorded in `perturbations_skipped` rather than silently counted as tested. +- `run_all.py` hardened — JSON scalar stdout lines no longer abort the pipeline, per-step return codes are tracked, the as-used speedup derivation guards both sides, and duplicate mode keys warn instead of silently overwriting. +- ge_arrow's `check_equivalence.py` writes `equivalence_x64.json` under `JAX_ENABLE_X64` instead of clobbering the as-shipped results. +- ge_arrow static metrics double-counted concept-token hits via a duplicated pattern (informational metric; 110 → 105). +- markov_asset's `statements_for_one_asset` renamed to `statements_for_one_result` to match the evidence-template vocabulary (values unchanged). +- Two files that were CRLF (`references/EVALUATION_FRAMEWORK.md`, the ge_arrow report) are normalized to LF, so a future one-line edit no longer renders as a whole-file diff. + +### benchmark 0.1.0 — 2026-07-07 + +First release: the plugin appears in the marketplace with a documented but not yet runnable evaluation procedure — a v0 outline skill, no executable scripts. + +- `/benchmark:eval-py-acceleration` — a v0 outline of the acceleration-review procedure: the five steps (equivalence check, static metrics, as-used benchmark, seven-dimension scoring, report), the seven weights (readability 0.25 deliberately above efficiency 0.15), the verdict bands, and the two calibration anchors (aiyagari Bellman ~25x faster as-used = HIGH; ge_arrow ~45x slower as-used = LOW). +- The guiding principle a user is meant to apply: lectures are teaching materials first, so "uses JAX" is never a goal in itself. +- `scripts/README.md` listing the eight measurement scripts still to be collected from [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717). + +--- + +**There is no 0.2.0.** It existed on a branch inside [#5](https://github.com/QuantEcon/skills/pull/5) and was superseded within the same pull request; because the repo squash-merges, `main` went 0.1.0 → 0.3.0 in one commit and 0.2.0 was never published. Nothing is missing from this file. + +## Historical — the `audit` plugin (folded into `qe` at 0.7.0) + +Released 2026-07-27 to 2026-08-07 as the separate `audit` plugin; its skill lives on as `/qe:audit-issues`. Frozen as shipped, same convention as above (tags `audit--v0.2.0` etc.). + +### audit 0.2.0 — 2026-08-07 + +The two severity-1 defects from the first measured run, which are the same defect at different altitudes: an audit's own record claiming more than it can support. + +**Changed** + +- **`[verified]` now requires evidence reachable from the ref the audit named.** Doctrine §2 is the single statement of it — §1 rule 1 no longer carries its own copy of the accepted-forms list, which is how the two drifted apart in the first place — and it covers every citation form — a commit must be an ancestor of the baseline ref, a `file:line` must be that line *on the ref* rather than in the working tree, a PR must be merged into it — and `/audit:issues` runs `git merge-base --is-ancestor ` before tagging a commit citation. Run 1's headline finding cited a commit that is real, does touch the file, and exists only on an unmerged branch, while the report's header said it had verified against `main`. A citation that resolves for its author and not for its reader is worse than an untagged claim, because the tag is what invited the trust. Evidence that genuinely lives off-ref stays citable — as the open PR it is, tagged `[stated]` or `[inferred]`. +- **Phase 2 checkpoints both of its passes.** `findings.md` now carries `## Open` and `## Closed` sections, and the resume rule partitions `issues.json` by state and resumes each side independently at the lowest number with no entry. Run 1 wrote only the 56 open issues to the checkpoint and sent the 62 closed ones straight to the catalog, so a run interrupted during the closed pass would have re-verified all 62 from scratch while reporting itself complete — the single-block resume rule shipped in 0.1.2 could not see the difference. + +### audit 0.1.4 — 2026-08-03 + +Doctrine §4's rule survived the first measured run; its justification did not. The section is re-derived from what that run actually produced, and the cost figures the skill quotes are replaced with measured ones. + +**Changed** + +- Doctrine §4 is renamed from "Surviving a long run" to "Checkpointing", and rests on three reasons that hold at any duration rather than on the claim that audits outlive sessions. The first measured run refuted that claim outright — 230 items in 22 minutes, with no context exhaustion, rate limit or sleeping machine in play. The strongest replacement reason is checkable: the per-item log is what the final enumeration is assembled *from*, and what a reviewer counts the coverage numbers against. +- `/audit:issues` no longer describes itself as "long-running by design — a hundred-issue repo is a multi-hour run". It now quotes the measured cost: roughly **10 seconds per open issue**, with a 230-item tracker carrying 56 open issues taking 22 minutes. Cost tracks open issues needing verification rather than total items, so a large tracker with a small open set is cheaper than a small one with a large set. +- The cost figures are stated so the two measures cannot be confused. Previously a reader met "roughly 10 seconds per open issue" beside "a 230-item tracker with 56 open took 22 minutes" and could not reconcile them — 56 × 10 s is 9 minutes, not 22. The 10-second rate is phase 2 alone; 22 minutes is end to end, and the remaining phases are largely fixed. Both numbers now say which question they answer. +- Checkpoint artifacts are named where they carry evidence rather than at every phase boundary out of symmetry — a checkpoint written and superseded minutes later without ever being read earns nothing. + +### audit 0.1.3 — 2026-08-03 + +**Added** + +- This changelog. + +**Fixed** + +- `/audit:issues`'s frontmatter `description` was an unquoted YAML plain scalar containing `Read-only: it recommends…`. A `: ` inside a plain scalar is a parse error, so a strict loader drops the skill's metadata rather than reading it, and `claude plugin validate` rejects the file outright. The value is now quoted. Nothing about the procedure changed. + +### audit 0.1.2 — 2026-07-28 + +Resolves the contradiction that told an audit to write its bundle into the repo it promised not to touch: the boundary is now mutation, not writing, and the skill says exactly where to put its working directory so a run leaves `git status` clean. + +**Added** + +- Discovery-ordered working-directory selection, taken from contact with a real repo: prefer a location the repo already ignores (`.dev/scratch/audit-/` in QuantEcon repos, where `.dev/scratch/*` is already gitignored), fall back to an untracked `.audit/-/` at the checkout root, then to somewhere outside the checkout entirely. Which one was used goes in the report's method section. +- Doctrine §3 now says explicitly that a run may write its own working directory, including inside the audited checkout — provided the directory stays untracked and nothing is added to `.gitignore`, since that would itself be an edit to a tracked file. + +**Changed** + +- Doctrine §3 narrowed from "no branch or file changes in the audited repo" to what it was always protecting — content and history: no commits, no pushes, no branches, no edits to tracked files. Mutation, not writing, is the boundary. +- `deliverables.md` states the split: writing the bundle is the audit's job, committing or publishing it is a human step taken after reading it. + +**Fixed** + +- The read-only/working-directory contradiction the plugin carried since 0.1.0 — §3 forbade file changes in the audited repo while `deliverables.md` made that repo's own notes system the bundle's first-choice destination, and 0.1.1's default `--out` wrote there too. A run following the docs literally could not satisfy both. + +### audit 0.1.1 — 2026-07-28 + +An interrupted run can actually be resumed: the intermediate artifacts now have names and locations, phase 2 appends per item instead of writing at the end, and the bundle shrinks to fit a small tracker. + +**Added** + +- A stated working-directory layout under `--out` (`.audit/-/` by convention): `snapshot/` from phase 1, `findings.md` from phase 2, `links.md` from phase 3, and the delivered `01-…`/`02-…`/`03-…`/`README.md` bundle from phase 4. Previously phases 2 and 3 produced "per-item findings" and "the cluster map" with no filename and no location, so resuming worked only if two sessions independently invented the same file. +- A stated resume rule: on restart, read `findings.md` and resume at the lowest number in `issues.json` with no entry, re-verifying the last entry rather than trusting a possibly truncated write. +- `meta.json` records `fetched_by`, the account the snapshot was taken as — which matters because visibility on the org's private repos is per-account. + +**Changed** + +- Phase 2 appends each item's finding to `findings.md` as it is verified, in the catalog entry format, so phase 4 assembles the catalog instead of re-deriving it. +- The bundle scales to the tracker: below roughly 30 open issues, fold the catalog and the link graph into the report, keep the `README.md` index, and say which shape was used in the coverage statement. Four unconditional documents forced three files of padding on a small tracker, and padding makes a report less checkable. +- Doctrine §4 now states the general rule: a checkpoint owes a findable name and incremental writes, or it is a claim about resumability rather than the property itself. + +**Fixed** + +- `meta["authenticated"]` is removed, not deprecated. It could only ever be `true` (preflight exits on every unauthenticated path), so it was a provenance field carrying no evidence — in the plugin whose doctrine is that every claim carries its evidence class. **Anything reading that field must switch to `fetched_by`.** +- An interrupted phase 2 now loses one item rather than the whole phase — it was the phase specified to write on completion, and the phase a hundred-item run dies inside rather than between. + +### audit 0.1.0 — 2026-07-27 + +First release. `/audit:issues` sweeps an entire GitHub tracker — open and closed — verifies each item against the code rather than the thread, tiers the open set into the repo's existing plan, and delivers an evidence-cited report bundle, without ever touching the tracker. + +- `/audit:issues ` — a whole-tracker audit in five phases (snapshot, per-item verification, cross-link graph, tiered report, coverage self-audit). The four runbook fields (plan anchor, tier scheme, repo type, notes system) are optional arguments with documented discovery, so the usual invocation is just the repo. +- A deterministic snapshot step, `scripts/fetch_tracker.py OWNER/REPO --out `: every issue and PR in any state with full comment threads (and PR reviews, and `closingIssuesReferences`) in two `gh` round trips, written as `meta.json`, `issues.json`, `prs.json`, `coverage.json`. Closed threads cost nothing extra to read, and the snapshot freezes the audit's point in time so "events after the snapshot" is a stated property of the report instead of an unnoticed gap. +- `coverage.json` reconciliation: captured items against the number sequence `1..max`, discussion counts split open/closed, and an explicit truncation flag when a stream returns exactly at `--limit` (default 1000) — a case indistinguishable from truncation, so it is surfaced rather than swallowed. PR review bodies count toward captured discussion, not just comments: on the example repo, closed PRs carried 374 reviews against 28 comments. +- Snapshot files are written in issue/PR number order, so two runs over an unchanged tracker are byte-identical and a re-fetch diffs down to what actually changed. +- Thread payloads are shape-asserted at capture, so a `gh` build returning counts instead of lists fails by name at the point of capture rather than crashing later or silently under-reporting threads while the report still claims thread-completeness. +- Preflight that refuses to start without `gh` and an authenticated account, because the anonymous API is 60 req/h per IP and returns nothing at all for the org's private repos. +- Plugin-level method shared by every future audit skill: `references/doctrine.md` (trust rules, evidence classes `[verified]`/`[stated]`/`[inferred]`, the read-only boundary, checkpointing, the coverage self-audit), `references/quantecon-context.md` (repo types, label ownership, the cross-repo graph, access, and the caveat that an HTML-reconstructed thread may start mid-conversation), and `references/deliverables.md` (what an audit owes its reader, and where a bundle may land). +- QuantEcon-specific triage judgement: tier by repo type (a build break in a lecture repo and a consumer-visible change in an action repo outrank thread activity), check sibling repos before concluding, leave label application to `qe`, and keep GitHub closing keywords out of drafted cross-repo references so drafted text cannot close an upstream item when someone posts it. +- The four-document bundle, the five phases and "produces a bundle" are stated as a worked example rather than a requirement, after a single execution. What an audit owes its reader — coverage statement, evidence tag per claim, recommendations marked as proposals, drafted comments marked unsent, a date and a named snapshot — stays mandatory and presumes no file count. diff --git a/qe/references/audit/README.md b/qe/references/audit/README.md new file mode 100644 index 0000000..c5d936e --- /dev/null +++ b/qe/references/audit/README.md @@ -0,0 +1,62 @@ +# The audit skills + +Bulk audits of a QuantEcon repository. Where `qe`'s author-facing skills serve one lecture and `/qe:benchmark` evaluates one conversion, this family sweeps a whole portfolio — every issue, every PR, a whole codebase, a whole translated series — and delivers a report. Shipped in the `qe` plugin; until qe 0.7.0 this was the separate `audit` plugin. + +## Skills + +| Skill | Audits | Status | +|---|---|---| +| [`/qe:audit-issues`](../../skills/audit-issues/SKILL.md) | Every issue, open and closed: status verified against the code, tiered into the repo's plan | runbook landed | +| `/qe:audit-prs` | Every open PR: does it solve a real issue, is it mergeable, what should the review say | candidate | +| `/qe:audit-tech-debt` | A codebase's accumulated debt, with a filing-ready issue catalog | candidate | +| `/qe:audit-translations` | Parity between a source series and a translation (`lecture-python.myst` ↔ `lecture-python.zh-cn`) | candidate | + +Only the first is written. The rest are candidates, tracked in [issue #12](https://github.com/QuantEcon/skills/issues/12) — each still needs the evidence a skill here normally carries before anyone writes it. Shipping one first is the point: the shared method gets proven against a real procedure before more are built on top of it. + +## What belongs here + +Two tests: + +1. **Bulk** — it sweeps a portfolio, not an item. Reviewing one PR's technical quality is not an audit; reviewing all of them is. +2. **Read-only** — it observes and reports. No skill here mutates a tracker, a branch, or a file in the audited repo ([doctrine §3](doctrine.md#3-read-only-boundary)). + +The read/write line is the one that matters, and it is deliberate: it mirrors the org's own automation split, where the family boundary *is* the permission boundary. Anything that acts on findings — filing the catalog as issues, posting the drafted comments, applying labels — is a separate human-invoked step, which is what makes this family safe to point at any repo and safe to run headlessly. (Consolidation into `qe` softened the old plugin-boundary framing — the enable unit is now the whole plugin — but the read-only rule itself is unchanged and doctrine §3 still enforces it per skill.) + +An audit also produces a written report rather than a chat answer, since the point is something a reader can check later. That says nothing about how long it is or how many files it takes — see [deliverables.md](deliverables.md), which describes what `/qe:audit-issues` produces without requiring the next skill to match it. + +## Shared references + +Skills are thin; the method lives once, shared by the family. + +| Document | Owns | +|---|---| +| [doctrine.md](doctrine.md) | Trust rules, evidence classes, read-only boundary, checkpointing, coverage self-audit | +| [quantecon-context.md](quantecon-context.md) | Repo types, label ownership, the cross-repo graph, notes-system discovery, access | +| [deliverables.md](deliverables.md) | What an audit owes its reader, where reports may land, and the `/qe:audit-issues` bundle as a worked example | +| [scripts/audit/](../../scripts/audit/) | Deterministic fetch machinery | + +## Running one + +``` +/qe:audit-issues QuantEcon/action-translation +``` + +Audits work from a frozen snapshot and checkpoint to disk as they go, so an interrupted run resumes rather than restarting — and every number in the report refers to one point in time. + +**Budget tens of minutes, not hours.** The first measured run covered a 230-item tracker in **22 minutes** end to end ([record](https://github.com/QuantEcon/skills/blob/main/reviews/audit-run-action-translation-2026-07-28.md)). What scales is the **open issue** count, not the item count: phase 2 verifies the open set at about 10 seconds each — 9 of those 22 minutes for 56 issues — while the remaining phases are largely fixed. So a 1000-item repo with a small open set is cheaper than a 300-item repo with a large one. That is one data point; a repo whose issues need deeper code archaeology will run slower per issue. + +Headless runs work the same way: + +```yaml +- uses: anthropics/claude-code-action@v1 + with: + plugin_marketplaces: "https://github.com/QuantEcon/skills.git" + plugins: "qe@quantecon" + prompt: "/qe:audit-issues QuantEcon/action-translation" +``` + +## A note on naming + +Inside the `qe` namespace the family reads as `/qe:audit-` — `/qe:audit-issues`, `/qe:audit-translations`. The `audit-` stem keeps the family greppable in a flat skill list ([#43](https://github.com/QuantEcon/skills/issues/43)), and each name still reads as a command. (Before consolidation the stem was the plugin: `/audit:issues`.) + +`audit` was chosen over `review` for the same reason the family excludes single-item work: `review` is already the per-item word here (the benchmark skill's review mode, and PR review generally), so an audit named `review-prs` would sit one keystroke from reviewing one PR. `audit` also matches QEP-3's `audit-` repo prefix and already connotes observe-and-report, which is the boundary this family enforces. diff --git a/audit/references/deliverables.md b/qe/references/audit/deliverables.md similarity index 90% rename from audit/references/deliverables.md rename to qe/references/audit/deliverables.md index 51e513b..b682f0a 100644 --- a/audit/references/deliverables.md +++ b/qe/references/audit/deliverables.md @@ -1,6 +1,6 @@ # Reporting -What an audit hands back. The four-document bundle below is what `/audit:issues` produces; it is a worked example, not a contract every skill in this plugin has to satisfy. A skill should produce what its subject actually needs — a debt audit's output is a filing-ready catalog, a parity audit's is a divergence list, and a small audit may reasonably be a single document. Reach for the bundle where it fits, and don't manufacture a section to fill a slot. +What an audit hands back. The four-document bundle below is what `/qe:audit-issues` produces; it is a worked example, not a contract every skill in this plugin has to satisfy. A skill should produce what its subject actually needs — a debt audit's output is a filing-ready catalog, a parity audit's is a divergence list, and a small audit may reasonably be a single document. Reach for the bundle where it fits, and don't manufacture a section to fill a slot. What every audit owes its reader is much smaller, and is in the last section. @@ -14,7 +14,7 @@ What every audit owes its reader is much smaller, and is in the last section. Confirm the destination before writing, and name it in the report. Writing the bundle there is the audit's job; **committing or publishing it is not** — that is a human step, taken after reading it ([doctrine §3](doctrine.md#3-read-only-boundary)). Private inputs stay in private destinations: a `project-*` repo's contents must not be summarised into a public one. -## The `/audit:issues` bundle +## The `/qe:audit-issues` bundle Four documents, which suit a whole-tracker review because it has an argument to make, a long enumeration to carry, and a graph worth drawing separately. Another audit may need two of these, or none. diff --git a/audit/references/doctrine.md b/qe/references/audit/doctrine.md similarity index 95% rename from audit/references/doctrine.md rename to qe/references/audit/doctrine.md index df3a099..c67d865 100644 --- a/audit/references/doctrine.md +++ b/qe/references/audit/doctrine.md @@ -61,9 +61,9 @@ How a skill divides itself into phases is its own business — the division belo Two properties separate a checkpoint from a claim about one, and a skill that promises resumability owes both. The artifact needs a **name the next session can find without guessing** — an unnamed intermediate is only resumable if two sessions independently invent the same file. And a phase that iterates over many items must **append as it works, not write when it finishes**, because the phase long enough to be worth checkpointing is the phase a run dies *inside*. Output that exists only on completion is no checkpoint at all, exactly where one was needed. -Two things are worth doing whatever the division. **Fetch once, and fetch first**, deterministically, in [`../scripts/`](../scripts/) rather than in model judgement. That also **freezes the audit's point in time**: every later claim refers to the snapshot, so "events after the snapshot" becomes a stated property of the report rather than an unnoticed gap. Record the snapshot timestamp; never silently mix fresh API reads into a later phase. +Two things are worth doing whatever the division. **Fetch once, and fetch first**, deterministically, in [`../scripts/`](../../scripts/audit/) rather than in model judgement. That also **freezes the audit's point in time**: every later claim refers to the snapshot, so "events after the snapshot" becomes a stated property of the report rather than an unnoticed gap. Record the snapshot timestamp; never silently mix fresh API reads into a later phase. -`/audit:issues` uses five phases, which suit an audit that must capture a whole tracker, check it item by item, and then write at length: +`/qe:audit-issues` uses five phases, which suit an audit that must capture a whole tracker, check it item by item, and then write at length: | Phase | Produces | Resumable from | |---|---|---| diff --git a/audit/references/quantecon-context.md b/qe/references/audit/quantecon-context.md similarity index 100% rename from audit/references/quantecon-context.md rename to qe/references/audit/quantecon-context.md diff --git a/benchmark/references/EVALUATION_FRAMEWORK.md b/qe/references/benchmark/EVALUATION_FRAMEWORK.md similarity index 92% rename from benchmark/references/EVALUATION_FRAMEWORK.md rename to qe/references/benchmark/EVALUATION_FRAMEWORK.md index bb40332..d4fce9a 100644 --- a/benchmark/references/EVALUATION_FRAMEWORK.md +++ b/qe/references/benchmark/EVALUATION_FRAMEWORK.md @@ -59,11 +59,11 @@ For each dimension we give (a) the metric(s) that quantify it, (b) the 1–5 anc ### How a score is computed (no hand-typed numbers) -The rubric is machine-encoded in [`scoring/rubric.py`](../scripts/scoring/rubric.py) so that **a score is a deterministic function of evidence**, applied identically to every lecture. The workflow — and the contract an AI skill follows — is: +The rubric is machine-encoded in [`scoring/rubric.py`](../../scripts/benchmark/scoring/rubric.py) so that **a score is a deterministic function of evidence**, applied identically to every lecture. The workflow — and the contract an AI skill follows — is: 1. **Measure** (per lecture, objective): run `/scripts/run_all.py` → `/results/*.json`. -2. **Record evidence** (per lecture): fill `/evidence.json` (schema: [`scoring/EVIDENCE_TEMPLATE.json`](../scripts/scoring/EVIDENCE_TEMPLATE.json)) — copy the measured numbers into the quantitative slots (noting the source file) and answer each structural checklist item true/false **with a citation to the diff**. This file, plus the measured results, is the *only* per-lecture input. -3. **Score** (shared, mechanical): `python scripts/scoring/score.py references/examples/` applies `rubric.py` and writes `/results/scorecard.json`, printing the derivation of every score. No score is ever written by hand; to change one you change a measured metric, a checklist answer, or the standard itself. +2. **Record evidence** (per lecture): fill `/evidence.json` (schema: [`scoring/EVIDENCE_TEMPLATE.json`](../../scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json)) — copy the measured numbers into the quantitative slots (noting the source file) and answer each structural checklist item true/false **with a citation to the diff**. This file, plus the measured results, is the *only* per-lecture input. +3. **Score** (shared, mechanical): `python scripts/benchmark/scoring/score.py references/benchmark/examples/` applies `rubric.py` and writes `/results/scorecard.json`, printing the derivation of every score. No score is ever written by hand; to change one you change a measured metric, a checklist answer, or the standard itself. **Quantitative dimensions (1, 2, 3, 6)** map a measured number to 1–5 via the threshold tables in the sections below (calibrated against two measured end points: the aiyagari Bellman pattern at 25× faster as-used, and the full `ge_arrow` lecture at 45× slower). @@ -183,7 +183,7 @@ measured over the lecture's *actual* sequence of solver calls, at its *actual* p | 2 | **< 0.8×** | measurably slower as used; stated goal not met, but correct & fixable | | 1 | < 0.8× **and** worse (wrong/unstable, or no fix path) | slower with no redemption | -> **HIGH (5) — MEASURED.** `aiyagari.md` is JAX on both branches (no NumPy baseline in-repo), so we benchmarked *its computational pattern* — the vectorised Bellman of `aiyagari.md:288-300` solved by value-function iteration on a `200×7` grid, then re-solved 20× as an equilibrium loop would. This is a **shared calibration of the efficiency threshold, not a per-lecture script** (`../scripts/calibration/bellman_bench.py`, results in `../scripts/calibration/bellman_bench.json`): +> **HIGH (5) — MEASURED.** `aiyagari.md` is JAX on both branches (no NumPy baseline in-repo), so we benchmarked *its computational pattern* — the vectorised Bellman of `aiyagari.md:288-300` solved by value-function iteration on a `200×7` grid, then re-solved 20× as an equilibrium loop would. This is a **shared calibration of the efficiency threshold, not a per-lecture script** (`../../scripts/benchmark/calibration/bellman_bench.py`, results in `../../scripts/benchmark/calibration/bellman_bench.json`): > > | | NumPy | JAX | speedup | > |---|--:|--:|--:| @@ -296,14 +296,14 @@ measured over the lecture's *actual* sequence of solver calls, at its *actual* p ```bash conda activate quantecon # jax 0.4.x, numpy 2.x, quantecon -python references/examples//scripts/run_all.py # measure → /results/*.json, +python references/benchmark/examples//scripts/run_all.py # measure → /results/*.json, # then apply the shared rubric -python scripts/scoring/score.py references/examples/ # (re)compute the scorecard alone +python scripts/benchmark/scoring/score.py references/benchmark/examples/ # (re)compute the scorecard alone ``` -`run_all.py` runs the measurement scripts (e.g. `check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, and lecture-specific ones) and finishes by invoking `scripts/scoring/score.py`, which reads `/evidence.json` and writes `/results/scorecard.json`. +`run_all.py` runs the measurement scripts (e.g. `check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, and lecture-specific ones) and finishes by invoking `scripts/benchmark/scoring/score.py`, which reads `/evidence.json` and writes `/results/scorecard.json`. -**To evaluate a *different* lecture** see the "Evaluate a new lecture" recipe in [`README.md`](../scripts/README.md): scaffold `/`, drop in `model_old.py` / `model_new.py`, adapt the measurement scripts, fill `evidence.json`, and run the two commands above. The framework, weights, thresholds, and checklists are lecture-independent; only the inputs change. +**To evaluate a *different* lecture** see the "Evaluate a new lecture" recipe in [`README.md`](../../scripts/benchmark/README.md): scaffold `/`, drop in `model_old.py` / `model_new.py`, adapt the measurement scripts, fill `evidence.json`, and run the two commands above. The framework, weights, thresholds, and checklists are lecture-independent; only the inputs change. ## 4. Limitations / honesty notes diff --git a/benchmark/README.md b/qe/references/benchmark/README.md similarity index 54% rename from benchmark/README.md rename to qe/references/benchmark/README.md index 2683b5e..5facd64 100644 --- a/benchmark/README.md +++ b/qe/references/benchmark/README.md @@ -1,6 +1,6 @@ -# benchmark plugin +# The benchmark skill -Evaluation tooling for QuantEcon lecture code rewrites — the question it answers is never "is JAX faster?" but **"does this implementation earn its place in this lecture?"** Lectures are teaching materials first and programs second; the plugin's rubric weights readability (0.25) above efficiency (0.15) on purpose. +Evaluation tooling for QuantEcon lecture code rewrites — the question it answers is never "is JAX faster?" but **"does this implementation earn its place in this lecture?"** Lectures are teaching materials first and programs second; the rubric weights readability (0.25) above efficiency (0.15) on purpose. Shipped as `/qe:benchmark` in the `qe` plugin; until qe 0.7.0 this was the separate `benchmark` plugin's `/benchmark:review-acceleration`. One skill, two modes — triage is the front door, since most questions arrive before any code is written: @@ -9,13 +9,13 @@ One skill, two modes — triage is the front door, since most questions arrive b | **Triage** | Is this lecture worth converting at all? | the existing lecture only | A recommendation with the binding constraint named | | **Review** | Did this conversion PR improve the lecture? | baseline + candidate implementations | A scored report with a merge recommendation | -Status: operational for workspace runs since v0.3.0, which wired the skill to the scoring engine — what shipped in which release is in [CHANGELOG.md](CHANGELOG.md), and remaining plan items are in [skills#4](https://github.com/QuantEcon/skills/issues/4). The system was developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). +Status: operational for workspace runs since benchmark 0.3.0, which wired the skill to the scoring engine — what shipped in which release is in the [qe changelog](../../CHANGELOG.md) (current stream, plus the retired `benchmark` plugin's entries as a historical section), and remaining plan items are in [skills#4](https://github.com/QuantEcon/skills/issues/4). The system was developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). ## Using the skill ``` -/benchmark:review-acceleration should we convert ? # triage mode -/benchmark:review-acceleration # review mode +/qe:benchmark should we convert ? # triage mode +/qe:benchmark # review mode ``` ### Triage mode — before any code is written @@ -27,7 +27,7 @@ The decision criteria are canonical in the manual's JAX style page — [when to 3. **Crossover comparison** — the lecture's problem sizes vs the warm crossover-n from the scaling data. 4. **Readability-cost forecast** — which concepts the conversion would force on readers (static args, `lax` carries, checkify, the float32/x64 distinction), against the prerequisite-concept bands. -Then the decision rule ([EVALUATION_FRAMEWORK.md §1](references/EVALUATION_FRAMEWORK.md)): the rubric weights readability above efficiency on purpose, so **a conversion that costs meaningful readability cannot break even on speed alone** — the weight algebra behind that lives in the framework, not here; it must also win on logic & design and ergonomics, and those structural wins are usually achievable in plain NumPy. +Then the decision rule ([EVALUATION_FRAMEWORK.md §1](EVALUATION_FRAMEWORK.md)): the rubric weights readability above efficiency on purpose, so **a conversion that costs meaningful readability cannot break even on speed alone** — the weight algebra behind that lives in the framework, not here; it must also win on logic & design and ergonomics, and those structural wins are usually achievable in plain NumPy. **Validation (2026-07-21):** triage applied blind (baseline-side data only) to the three known cases reproduces every known verdict: @@ -43,7 +43,7 @@ Scope limit, confirmed by the same test: triage predicts whether the prize is wo ### Review mode — what you get -The skill follows the measure → record-evidence → score contract ([scripts/README.md](scripts/README.md)): it extracts both implementations verbatim from the lecture's code cells, adapts the measurement templates, runs them, fills `evidence.json` with cited answers, and lets the engine compute the verdict — **no score is ever typed by hand**. The session shows the engine's derivation table (every score with the measured number and threshold band that produced it), and the final report follows the worked examples' format, leading with the decision: +The skill follows the measure → record-evidence → score contract ([scripts/benchmark/README.md](../../scripts/benchmark/README.md)): it extracts both implementations verbatim from the lecture's code cells, adapts the measurement templates, runs them, fills `evidence.json` with cited answers, and lets the engine compute the verdict — **no score is ever typed by hand**. The session shows the engine's derivation table (every score with the measured number and threshold band that produced it), and the final report follows the worked examples' format, leading with the decision: 1. **TL;DR** — the *full* verdict first (gate / no-conversion / sensitivity included), the decisive facts, and the weighted score alongside as candidate quality for the record 2. **Dimension table** — a verdict row, then weight / score / weighted contribution / one-line driver each, so the table still carries the decision when quoted on its own @@ -51,29 +51,29 @@ The skill follows the measure → record-evidence → score contract ([scripts/R 4. **Evidence by dimension** — `max|Δ|` in both dtype regimes, prerequisite-concept and docstring deltas, the as-used vs warm timing table, crossover-n, recompile audit 5. **Recommendation** — a must-fix list where each item is tagged with the dimension it lifts, plus where the score lands after fixes -See [references/examples/ge_arrow/ge_arrow_REPORT.md](references/examples/ge_arrow/ge_arrow_REPORT.md) (2.85/5, no-conversion; candidate band mixed/wash) and [references/examples/markov_asset/markov_asset_REPORT.md](references/examples/markov_asset/markov_asset_REPORT.md) (2.25/5, no-conversion + gated net regression) for complete real reports. Verdict bands, the v2 verdict gates / no-conversion rule / sensitivity stamp, weights, and scoring anchors: [references/EVALUATION_FRAMEWORK.md](references/EVALUATION_FRAMEWORK.md) §1–2. +See [examples/ge_arrow/ge_arrow_REPORT.md](examples/ge_arrow/ge_arrow_REPORT.md) (2.85/5, no-conversion; candidate band mixed/wash) and [examples/markov_asset/markov_asset_REPORT.md](examples/markov_asset/markov_asset_REPORT.md) (2.25/5, no-conversion + gated net regression) for complete real reports. Verdict bands, the v2 verdict gates / no-conversion rule / sensitivity stamp, weights, and scoring anchors: [EVALUATION_FRAMEWORK.md](EVALUATION_FRAMEWORK.md) §1–2. **The one rule to remember:** warm-only speedups are never the headline. The ge_arrow case measured 1.4–4.8× faster warm and **45× slower as-used** — the as-used number (fresh process, actual problem sizes, compile time included) decides the efficiency score. ## Manual usage (no skill) -The full recipe is in [scripts/README.md](scripts/README.md) ("Evaluating a new lecture"); quickstart from this directory: +The full recipe is in [scripts/benchmark/README.md](../../scripts/benchmark/README.md) ("Evaluating a new lecture"); quickstart from the plugin root (`qe/`): ```bash conda activate quantecon -python references/examples//scripts/run_all.py # measure + provenance stamp -# fill references/examples//evidence.json (numbers + cited yes/no answers) -python scripts/scoring/score.py references/examples/ +python references/benchmark/examples//scripts/run_all.py # measure + provenance stamp +# fill references/benchmark/examples//evidence.json (numbers + cited yes/no answers) +python scripts/benchmark/scoring/score.py references/benchmark/examples/ ``` -Sanity anchors: re-running either worked example must reproduce **2.85** / **2.25** (both now carrying the v2 **no-conversion** verdict; ge_arrow stamps *fragile*, markov_asset *robust-at-floor* — its verdict is already bottom-band, so nothing could move it downward). The synthetic fixtures in [`references/fixtures/`](references/fixtures/) reproduce alongside them; CI regenerates all three and fails on any diff. A step-by-step walkthrough of the whole procedure — with the ge_arrow reproduction as a checkable example — is [docs/tutorial-run-an-evaluation.md](https://github.com/QuantEcon/skills/blob/main/docs/tutorial-run-an-evaluation.md). +Sanity anchors: re-running either worked example must reproduce **2.85** / **2.25** (both now carrying the v2 **no-conversion** verdict; ge_arrow stamps *fragile*, markov_asset *robust-at-floor* — its verdict is already bottom-band, so nothing could move it downward). The synthetic fixtures in [`fixtures/`](fixtures/) reproduce alongside them; CI regenerates all three and fails on any diff. A step-by-step walkthrough of the whole procedure — with the ge_arrow reproduction as a checkable example — is [docs/tutorial-run-an-evaluation.md](https://github.com/QuantEcon/skills/blob/main/docs/tutorial-run-an-evaluation.md). ## Map | Path | What | |---|---| -| [skills/review-acceleration/](skills/review-acceleration/SKILL.md) | The skill (procedure, both modes) | -| [scripts/README.md](scripts/README.md) | Deterministic engine (`scripts/scoring/`): rubric, scorer, evidence template, provenance stamp | -| [scripts/calibration/](scripts/calibration/bellman_bench.py) | The shared HIGH-efficiency anchor (~24× ⇒ score 5) | -| [references/EVALUATION_FRAMEWORK.md](references/EVALUATION_FRAMEWORK.md) | The standard in prose — weights, anchors, checklists, verdict bands | -| [references/examples/](references/examples/README.md) | Two complete worked evaluations + the logic-check/provenance audit; the regression baseline | +| [skills/benchmark/](../../skills/benchmark/SKILL.md) | The skill (procedure, both modes) | +| [scripts/benchmark/README.md](../../scripts/benchmark/README.md) | Deterministic engine (`scripts/benchmark/scoring/`): rubric, scorer, evidence template, provenance stamp | +| [scripts/benchmark/calibration/](../../scripts/benchmark/calibration/bellman_bench.py) | The shared HIGH-efficiency anchor (~24× ⇒ score 5) | +| [EVALUATION_FRAMEWORK.md](EVALUATION_FRAMEWORK.md) | The standard in prose — weights, anchors, checklists, verdict bands | +| [examples/](examples/README.md) | Two complete worked evaluations + the logic-check/provenance audit; the regression baseline | diff --git a/benchmark/references/examples/README.md b/qe/references/benchmark/examples/README.md similarity index 94% rename from benchmark/references/examples/README.md rename to qe/references/benchmark/examples/README.md index 92a3e24..d5dc449 100644 --- a/benchmark/references/examples/README.md +++ b/qe/references/benchmark/examples/README.md @@ -1,6 +1,6 @@ # Reference examples — logic check and provenance -The two evaluations in this directory are the **canonical reference cases** for `/benchmark:review-acceleration`: they calibrate the rubric's thresholds, serve as the worked demonstrations of the method, and act as the **regression baseline** — any change to the scoring engine or the measurement templates must reproduce their scorecards. This document explains each example in detail and records the line-by-line logic review performed on 2026-07-21 (all scripts, all data files), so their accuracy is auditable rather than asserted. +The two evaluations in this directory are the **canonical reference cases** for `/qe:benchmark`: they calibrate the rubric's thresholds, serve as the worked demonstrations of the method, and act as the **regression baseline** — any change to the scoring engine or the measurement templates must reproduce their scorecards. This document explains each example in detail and records the line-by-line logic review performed on 2026-07-21 (all scripts, all data files), so their accuracy is auditable rather than asserted. **Review verdict: logic sound, methodology fair, every traceable number consistent with its source.** Known caveats are listed at the end — none changes either verdict. @@ -15,7 +15,7 @@ Each example directory is one self-contained evaluation of a lecture rewrite (Nu | `scripts/model_old.py`, `model_new.py` | Faithful extractions of the two implementations. Any deviation from the lecture source is disclosed in the module docstring and is itself an evaluation finding | | `scripts/*` (measurement) | Produce objective numbers into `results/*.json` | | `evidence.json` | The **only place judgement is recorded**: measured numbers copied into quantitative slots (with source), structural checklist items answered true/false with citations to the diff | -| `results/scorecard.json` | Computed by `../../scripts/scoring/score.py` from the evidence — **no score is ever typed by hand** | +| `results/scorecard.json` | Computed by `../../../scripts/benchmark/scoring/score.py` from the evidence — **no score is ever typed by hand** | | `_REPORT.md` | The human-readable verdict, written from scorecard + evidence | The measurement standard throughout: **as-used, fresh-process** — replay the lecture's actual call sequence at its actual problem sizes in a new interpreter, so JAX trace/compile time counts; warm numbers are reported alongside, never alone. @@ -90,7 +90,7 @@ correctness 1 (does not build) + readability 2 + efficiency 2 + logic 3 (capped) --- -## The shared HIGH anchor — `../../scripts/calibration/bellman_bench.py` +## The shared HIGH anchor — `../../../scripts/benchmark/calibration/bellman_bench.py` `aiyagari.md` is JAX on both branches, so the HIGH end of the efficiency scale is calibrated on its *computational pattern*: the vectorised Bellman operator on a 200×7 grid, solved by VFI (~397 iterations), then re-solved 20× as an equilibrium loop would — the regime JAX is built for. Fairness properties verified: x64 enabled; the NumPy baseline uses the same broadcast/vectorised algorithm (not a strawman); implementations agree to ~1e-14; cold timing uses `_clear_cache()`; the equilibrium loop includes exactly one compile. Result: **~25× faster as-used** → pins "≥3× → score 5", with ge_arrow's 0.022× anchoring the LOW end. diff --git a/benchmark/references/examples/ge_arrow/evidence.json b/qe/references/benchmark/examples/ge_arrow/evidence.json similarity index 96% rename from benchmark/references/examples/ge_arrow/evidence.json rename to qe/references/benchmark/examples/ge_arrow/evidence.json index 96bf7b0..cc81c42 100644 --- a/benchmark/references/examples/ge_arrow/evidence.json +++ b/qe/references/benchmark/examples/ge_arrow/evidence.json @@ -6,7 +6,7 @@ "base": "8cfba4c90ebc08d3e51718ee65246ac249305ce0", "head": "8c2d0d798d6ad7fb669351858f0e0d8e07659f6b" }, - "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/scoring/score.py references/examples/ (from the plugin root).", + "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/benchmark/scoring/score.py references/benchmark/examples/ (from the plugin root).", "quantitative": { "correctness": { diff --git a/benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md b/qe/references/benchmark/examples/ge_arrow/ge_arrow_REPORT.md similarity index 96% rename from benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md rename to qe/references/benchmark/examples/ge_arrow/ge_arrow_REPORT.md index 094db48..d741612 100644 --- a/benchmark/references/examples/ge_arrow/ge_arrow_REPORT.md +++ b/qe/references/benchmark/examples/ge_arrow/ge_arrow_REPORT.md @@ -1,6 +1,6 @@ # Evaluation Report — `ge_arrow.md`: NumPy (`main`) → JAX (`update_ge_arrow`) -Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the only code change on branch `update_ge_arrow`. All numbers below are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../scripts/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. +Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the only code change on branch `update_ge_arrow`. All numbers below are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../../scripts/benchmark/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. > **Rubric v2 note (2026-07-22).** Re-scored under rubric v2 (verdict gates, no-conversion, sensitivity stamp — see `reviews/`): the total is unchanged at **2.85/5**, but the headline verdict is now **no-conversion** — the baseline as-used total (0.035 s) is under the 1 s materiality floor and the candidate is slower as-used (0.022×), so this lecture should not be converted regardless of the candidate's polish (candidate band for the record: mixed/wash). Sensitivity stamp: **fragile** — flipping `good_algorithmic_choices` alone moves the candidate band to 3.00/net-positive, and flipping either correctness boolean drops it to a gated net regression. Derivation: `results/scorecard.json`. @@ -76,7 +76,7 @@ Per-regime detail explaining why: | λ-sweep (100 pts), as run once | 1.8 ms | 300 ms cold | **170× slower** | | λ-sweep warm | — | 0.37 ms | 4.8× faster *(never realized)* | -Scaling crossover (`benchmark.py`): NumPy and JAX-warm are even near **n≈10**; JAX wins **2–6×** for `n = 25…200`. **The lecture never exceeds n=3.** For calibration, the same machinery on the large, repeatedly-solved aiyagari pattern (shared `../../../scripts/calibration/bellman_bench.py`) is **25× faster** — a score-5 case. `ge_arrow`'s `0.022×` maps to **score 2** (< 0.8×, but correct and fixable). +Scaling crossover (`benchmark.py`): NumPy and JAX-warm are even near **n≈10**; JAX wins **2–6×** for `n = 25…200`. **The lecture never exceeds n=3.** For calibration, the same machinery on the large, repeatedly-solved aiyagari pattern (shared `../../../../scripts/benchmark/calibration/bellman_bench.py`) is **25× faster** — a score-5 case. `ge_arrow`'s `0.022×` maps to **score 2** (< 0.8×, but correct and fixable). ### 4 · Logic & design → **4/5** Genuine improvements, all verified in the diff: diff --git a/benchmark/references/examples/ge_arrow/results/benchmark.json b/qe/references/benchmark/examples/ge_arrow/results/benchmark.json similarity index 100% rename from benchmark/references/examples/ge_arrow/results/benchmark.json rename to qe/references/benchmark/examples/ge_arrow/results/benchmark.json diff --git a/benchmark/references/examples/ge_arrow/results/equivalence.json b/qe/references/benchmark/examples/ge_arrow/results/equivalence.json similarity index 100% rename from benchmark/references/examples/ge_arrow/results/equivalence.json rename to qe/references/benchmark/examples/ge_arrow/results/equivalence.json diff --git a/benchmark/references/examples/ge_arrow/results/scorecard.json b/qe/references/benchmark/examples/ge_arrow/results/scorecard.json similarity index 97% rename from benchmark/references/examples/ge_arrow/results/scorecard.json rename to qe/references/benchmark/examples/ge_arrow/results/scorecard.json index b167695..fac42f8 100644 --- a/benchmark/references/examples/ge_arrow/results/scorecard.json +++ b/qe/references/benchmark/examples/ge_arrow/results/scorecard.json @@ -122,5 +122,5 @@ } } ], - "_note": "Scores are computed by scripts/scoring/rubric.py from ge_arrow/evidence.json; do not edit by hand." + "_note": "Scores are computed by scripts/benchmark/scoring/rubric.py from ge_arrow/evidence.json; do not edit by hand." } \ No newline at end of file diff --git a/benchmark/references/examples/ge_arrow/results/static_metrics.json b/qe/references/benchmark/examples/ge_arrow/results/static_metrics.json similarity index 100% rename from benchmark/references/examples/ge_arrow/results/static_metrics.json rename to qe/references/benchmark/examples/ge_arrow/results/static_metrics.json diff --git a/benchmark/references/examples/ge_arrow/results/sweep.json b/qe/references/benchmark/examples/ge_arrow/results/sweep.json similarity index 100% rename from benchmark/references/examples/ge_arrow/results/sweep.json rename to qe/references/benchmark/examples/ge_arrow/results/sweep.json diff --git a/benchmark/references/examples/ge_arrow/scripts/as_used_total.py b/qe/references/benchmark/examples/ge_arrow/scripts/as_used_total.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/as_used_total.py rename to qe/references/benchmark/examples/ge_arrow/scripts/as_used_total.py diff --git a/benchmark/references/examples/ge_arrow/scripts/benchmark.py b/qe/references/benchmark/examples/ge_arrow/scripts/benchmark.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/benchmark.py rename to qe/references/benchmark/examples/ge_arrow/scripts/benchmark.py diff --git a/benchmark/references/examples/ge_arrow/scripts/check_equivalence.py b/qe/references/benchmark/examples/ge_arrow/scripts/check_equivalence.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/check_equivalence.py rename to qe/references/benchmark/examples/ge_arrow/scripts/check_equivalence.py diff --git a/benchmark/references/examples/ge_arrow/scripts/cold_start.py b/qe/references/benchmark/examples/ge_arrow/scripts/cold_start.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/cold_start.py rename to qe/references/benchmark/examples/ge_arrow/scripts/cold_start.py diff --git a/benchmark/references/examples/ge_arrow/scripts/model_new.py b/qe/references/benchmark/examples/ge_arrow/scripts/model_new.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/model_new.py rename to qe/references/benchmark/examples/ge_arrow/scripts/model_new.py diff --git a/benchmark/references/examples/ge_arrow/scripts/model_old.py b/qe/references/benchmark/examples/ge_arrow/scripts/model_old.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/model_old.py rename to qe/references/benchmark/examples/ge_arrow/scripts/model_old.py diff --git a/benchmark/references/examples/ge_arrow/scripts/run_all.py b/qe/references/benchmark/examples/ge_arrow/scripts/run_all.py similarity index 89% rename from benchmark/references/examples/ge_arrow/scripts/run_all.py rename to qe/references/benchmark/examples/ge_arrow/scripts/run_all.py index e36add2..b650641 100644 --- a/benchmark/references/examples/ge_arrow/scripts/run_all.py +++ b/qe/references/benchmark/examples/ge_arrow/scripts/run_all.py @@ -32,7 +32,7 @@ def median(xs): ("As-used total (numpy)", "as_used_total.py numpy", "as_used.json"), ("As-used total (jax)", "as_used_total.py jax", "as_used.json"), # HIGH-end efficiency calibration is shared, not lecture-specific: - # see scripts/calibration/bellman_bench.py + # see scripts/benchmark/calibration/bellman_bench.py ] collected = {} @@ -103,16 +103,16 @@ def median(xs): # Shared engine location: the installed plugin root when the skill drives an # evaluation from a user workspace; falls back to this repo's layout. PLUGIN = (os.environ.get("CLAUDE_PLUGIN_ROOT") - or os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR)))) + or os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR))))) -# Provenance stamp (shared: scripts/scoring/env_stamp.py — the seed of the +# Provenance stamp (shared: scripts/benchmark/scoring/env_stamp.py — the seed of the # QuantEcon/meta#335 result + environment-descriptor schema). Failed step # titles are recorded so a partial run cannot claim full provenance. -subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "env_stamp.py"), +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "benchmark", "scoring", "env_stamp.py"), LEC_DIR] + failed, check=False) # Scoring is shared across lectures: fill ../evidence.json from the results -# above, then apply the common rubric (scripts/scoring/rubric.py) via the engine. +# above, then apply the common rubric (scripts/benchmark/scoring/rubric.py) via the engine. print("\n" + "=" * 70 + "\n== Scorecard (shared rubric)\n" + "=" * 70) -subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "score.py"), +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "benchmark", "scoring", "score.py"), LEC_DIR], check=False) diff --git a/benchmark/references/examples/ge_arrow/scripts/static_metrics.py b/qe/references/benchmark/examples/ge_arrow/scripts/static_metrics.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/static_metrics.py rename to qe/references/benchmark/examples/ge_arrow/scripts/static_metrics.py diff --git a/benchmark/references/examples/ge_arrow/scripts/sweep_bench.py b/qe/references/benchmark/examples/ge_arrow/scripts/sweep_bench.py similarity index 100% rename from benchmark/references/examples/ge_arrow/scripts/sweep_bench.py rename to qe/references/benchmark/examples/ge_arrow/scripts/sweep_bench.py diff --git a/benchmark/references/examples/markov_asset/evidence.json b/qe/references/benchmark/examples/markov_asset/evidence.json similarity index 97% rename from benchmark/references/examples/markov_asset/evidence.json rename to qe/references/benchmark/examples/markov_asset/evidence.json index bf47b8f..d4a926b 100644 --- a/benchmark/references/examples/markov_asset/evidence.json +++ b/qe/references/benchmark/examples/markov_asset/evidence.json @@ -6,7 +6,7 @@ "base": "8cfba4c90ebc08d3e51718ee65246ac249305ce0", "head": "533f572d6d5abf3660b2cae93d667d9df9c931e0" }, - "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/scoring/score.py references/examples/ (from the plugin root).", + "_how": "Quantitative values are copied from results/*.json (source noted). Structural criteria are yes/no facts about the diff, each with a citation. Scores are computed by scoring/rubric.py — run: python scripts/benchmark/scoring/score.py references/benchmark/examples/ (from the plugin root).", "quantitative": { "correctness": { diff --git a/benchmark/references/examples/markov_asset/markov_asset_REPORT.md b/qe/references/benchmark/examples/markov_asset/markov_asset_REPORT.md similarity index 98% rename from benchmark/references/examples/markov_asset/markov_asset_REPORT.md rename to qe/references/benchmark/examples/markov_asset/markov_asset_REPORT.md index cee9acc..e41357f 100644 --- a/benchmark/references/examples/markov_asset/markov_asset_REPORT.md +++ b/qe/references/benchmark/examples/markov_asset/markov_asset_REPORT.md @@ -2,7 +2,7 @@ > **Erratum (2026-07-21).** This report states that "the lecture does not build as shipped." Design review later established this is false *as worded*: executing the notebook's cells **in order** completes cleanly (and the PR's preview CI passes), because earlier cells bind a global `err` immediately before `call_option_jit` is first traced, so the stray `err.throw()` inside `call_option` resolves to that stale, already-checked object and silently does nothing. The defect is real and arguably worse than a crash: the stale-global masking means the checkify stability validation inside `call_option` is **never actually performed** in the shipped lecture — on the model whose spectral radius sits 0.002 below the stability bound — and any reader copying the function into a clean namespace hits the `NameError` this report describes. The evaluation's `builds: false` refers to fresh-process/clean-namespace execution of the extracted implementation (the system's declared measurement regime); the verdict and must-fix list stand. Nothing has been posted to [lecture-python.myst#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) — the PR will receive one authoritative evaluation once the rubric-v2 revision lands and the skill has run the full protocol. See the design-review documents in `reviews/`. -Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the NumPy→JAX conversion of `markov_asset.md`. All numbers are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3, quantecon) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../scripts/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. +Applies the system in [`../../EVALUATION_FRAMEWORK.md`](../../EVALUATION_FRAMEWORK.md) to the NumPy→JAX conversion of `markov_asset.md`. All numbers are reproduced by `scripts/run_all.py` (CPU, jax 0.4.35, numpy 2.1.3, quantecon) into `results/`. Every dimension score is **computed from [`evidence.json`](evidence.json) by the shared rubric** (`../../../../scripts/benchmark/scoring/rubric.py`) — see `results/scorecard.json` for the derivation. > **Rubric v2 note (2026-07-22).** Re-scored under rubric v2 (verdict gates, no-conversion, sensitivity stamp — see `reviews/`): the total is unchanged at **2.25/5** and the candidate band remains *net regression* — now **gated**: correctness 1 caps the verdict independent of the weighted sum, and the logic&design bug-cap is derived from the correctness evidence rather than a hand-set boolean. The headline verdict is now **no-conversion** (baseline as-used 0.18 s, 0.17× slowdown). Sensitivity stamp: **robust** across all 29 single-input perturbations — the design review's one-concept band flip (2.25→2.50) no longer changes the verdict because the correctness gate holds it at net regression. Derivation: `results/scorecard.json`. diff --git a/benchmark/references/examples/markov_asset/results/equivalence_x64_False.json b/qe/references/benchmark/examples/markov_asset/results/equivalence_x64_False.json similarity index 100% rename from benchmark/references/examples/markov_asset/results/equivalence_x64_False.json rename to qe/references/benchmark/examples/markov_asset/results/equivalence_x64_False.json diff --git a/benchmark/references/examples/markov_asset/results/equivalence_x64_True.json b/qe/references/benchmark/examples/markov_asset/results/equivalence_x64_True.json similarity index 100% rename from benchmark/references/examples/markov_asset/results/equivalence_x64_True.json rename to qe/references/benchmark/examples/markov_asset/results/equivalence_x64_True.json diff --git a/benchmark/references/examples/markov_asset/results/scaling.json b/qe/references/benchmark/examples/markov_asset/results/scaling.json similarity index 100% rename from benchmark/references/examples/markov_asset/results/scaling.json rename to qe/references/benchmark/examples/markov_asset/results/scaling.json diff --git a/benchmark/references/examples/markov_asset/results/scorecard.json b/qe/references/benchmark/examples/markov_asset/results/scorecard.json similarity index 97% rename from benchmark/references/examples/markov_asset/results/scorecard.json rename to qe/references/benchmark/examples/markov_asset/results/scorecard.json index c990c9d..73615e2 100644 --- a/benchmark/references/examples/markov_asset/results/scorecard.json +++ b/qe/references/benchmark/examples/markov_asset/results/scorecard.json @@ -101,5 +101,5 @@ } } ], - "_note": "Scores are computed by scripts/scoring/rubric.py from markov_asset/evidence.json; do not edit by hand." + "_note": "Scores are computed by scripts/benchmark/scoring/rubric.py from markov_asset/evidence.json; do not edit by hand." } \ No newline at end of file diff --git a/benchmark/references/examples/markov_asset/results/static_metrics.json b/qe/references/benchmark/examples/markov_asset/results/static_metrics.json similarity index 100% rename from benchmark/references/examples/markov_asset/results/static_metrics.json rename to qe/references/benchmark/examples/markov_asset/results/static_metrics.json diff --git a/benchmark/references/examples/markov_asset/scripts/as_used_total.py b/qe/references/benchmark/examples/markov_asset/scripts/as_used_total.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/as_used_total.py rename to qe/references/benchmark/examples/markov_asset/scripts/as_used_total.py diff --git a/benchmark/references/examples/markov_asset/scripts/benchmark.py b/qe/references/benchmark/examples/markov_asset/scripts/benchmark.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/benchmark.py rename to qe/references/benchmark/examples/markov_asset/scripts/benchmark.py diff --git a/benchmark/references/examples/markov_asset/scripts/check_equivalence.py b/qe/references/benchmark/examples/markov_asset/scripts/check_equivalence.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/check_equivalence.py rename to qe/references/benchmark/examples/markov_asset/scripts/check_equivalence.py diff --git a/benchmark/references/examples/markov_asset/scripts/model_new.py b/qe/references/benchmark/examples/markov_asset/scripts/model_new.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/model_new.py rename to qe/references/benchmark/examples/markov_asset/scripts/model_new.py diff --git a/benchmark/references/examples/markov_asset/scripts/model_old.py b/qe/references/benchmark/examples/markov_asset/scripts/model_old.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/model_old.py rename to qe/references/benchmark/examples/markov_asset/scripts/model_old.py diff --git a/benchmark/references/examples/markov_asset/scripts/run_all.py b/qe/references/benchmark/examples/markov_asset/scripts/run_all.py similarity index 90% rename from benchmark/references/examples/markov_asset/scripts/run_all.py rename to qe/references/benchmark/examples/markov_asset/scripts/run_all.py index 633e8e7..f96795e 100644 --- a/benchmark/references/examples/markov_asset/scripts/run_all.py +++ b/qe/references/benchmark/examples/markov_asset/scripts/run_all.py @@ -93,18 +93,18 @@ def median(xs): # Shared engine location: the installed plugin root when the skill drives an # evaluation from a user workspace; falls back to this repo's layout. PLUGIN = (os.environ.get("CLAUDE_PLUGIN_ROOT") - or os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR)))) + or os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(LEC_DIR))))) -# Provenance stamp (shared: scripts/scoring/env_stamp.py — the seed of the +# Provenance stamp (shared: scripts/benchmark/scoring/env_stamp.py — the seed of the # QuantEcon/meta#335 result + environment-descriptor schema). Failed step # titles are recorded so a partial run cannot claim full provenance. -subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "env_stamp.py"), +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "benchmark", "scoring", "env_stamp.py"), LEC_DIR] + failed, check=False) # Scoring is shared across lectures: fill ../evidence.json from the results -# above, then apply the common rubric (scripts/scoring/rubric.py) via the engine. +# above, then apply the common rubric (scripts/benchmark/scoring/rubric.py) via the engine. print("\n" + "=" * 70 + "\n== Scorecard (shared rubric)\n" + "=" * 70) -subprocess.run([PY, os.path.join(PLUGIN, "scripts", "scoring", "score.py"), +subprocess.run([PY, os.path.join(PLUGIN, "scripts", "benchmark", "scoring", "score.py"), LEC_DIR], check=False) print("\nNote: to also get the x64 equivalence numbers, run:") diff --git a/benchmark/references/examples/markov_asset/scripts/smoke_test.py b/qe/references/benchmark/examples/markov_asset/scripts/smoke_test.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/smoke_test.py rename to qe/references/benchmark/examples/markov_asset/scripts/smoke_test.py diff --git a/benchmark/references/examples/markov_asset/scripts/static_metrics.py b/qe/references/benchmark/examples/markov_asset/scripts/static_metrics.py similarity index 100% rename from benchmark/references/examples/markov_asset/scripts/static_metrics.py rename to qe/references/benchmark/examples/markov_asset/scripts/static_metrics.py diff --git a/benchmark/references/fixtures/README.md b/qe/references/benchmark/fixtures/README.md similarity index 93% rename from benchmark/references/fixtures/README.md rename to qe/references/benchmark/fixtures/README.md index 0997a33..81a9fd5 100644 --- a/benchmark/references/fixtures/README.md +++ b/qe/references/benchmark/fixtures/README.md @@ -22,14 +22,14 @@ Both worked examples conclude *no-conversion* on sub-second baselines, take the The baseline is deliberately set above the 1 s materiality floor so the **no-conversion** verdict does not fire and mask paths 1–5 — that verdict is already covered by both worked examples. -The scoring rules themselves are defined once in [`../EVALUATION_FRAMEWORK.md`](../EVALUATION_FRAMEWORK.md) §1–2 and implemented in `../../scripts/scoring/rubric.py`; this file says only which of them the fixture pins. +The scoring rules themselves are defined once in [`../EVALUATION_FRAMEWORK.md`](../EVALUATION_FRAMEWORK.md) §1–2 and implemented in `../../../scripts/benchmark/scoring/rubric.py`; this file says only which of them the fixture pins. ## Changing a fixture The specific values are the test. If a rubric change makes a fixture's scorecard move, that is the check working — confirm the new behaviour is what you intended, then regenerate and commit: ```bash -python scripts/scoring/score.py references/fixtures/rubric_v2 +python scripts/benchmark/scoring/score.py references/benchmark/fixtures/rubric_v2 ``` Adjusting a fixture's inputs to restore an old scorecard defeats the point. If a path stops being reachable at all, delete the fixture and say why in the commit, rather than leaving one that no longer tests anything. diff --git a/benchmark/references/fixtures/rubric_v2/evidence.json b/qe/references/benchmark/fixtures/rubric_v2/evidence.json similarity index 100% rename from benchmark/references/fixtures/rubric_v2/evidence.json rename to qe/references/benchmark/fixtures/rubric_v2/evidence.json diff --git a/benchmark/references/fixtures/rubric_v2/results/scorecard.json b/qe/references/benchmark/fixtures/rubric_v2/results/scorecard.json similarity index 97% rename from benchmark/references/fixtures/rubric_v2/results/scorecard.json rename to qe/references/benchmark/fixtures/rubric_v2/results/scorecard.json index 5b315da..2892649 100644 --- a/benchmark/references/fixtures/rubric_v2/results/scorecard.json +++ b/qe/references/benchmark/fixtures/rubric_v2/results/scorecard.json @@ -108,5 +108,5 @@ } } ], - "_note": "Scores are computed by scripts/scoring/rubric.py from rubric_v2/evidence.json; do not edit by hand." + "_note": "Scores are computed by scripts/benchmark/scoring/rubric.py from rubric_v2/evidence.json; do not edit by hand." } \ No newline at end of file diff --git a/qe/scripts/README.md b/qe/scripts/README.md index 4eac613..d6976ee 100644 --- a/qe/scripts/README.md +++ b/qe/scripts/README.md @@ -1,5 +1,7 @@ # qe plugin — scripts +Each skill family's deterministic machinery lives in its own subdirectory: [`benchmark/`](benchmark/README.md) (the scoring engine and calibration behind `/qe:benchmark`) and [`audit/`](audit/README.md) (the tracker-snapshot fetcher behind `/qe:audit-issues`). `fetch-copilot.sh` predates that layout and stays at the top level, where `/qe:copilot-review` already points. + ## `fetch-copilot.sh` Dumps GitHub Copilot's review of a pull request — the overview, then every inline comment with the ID to reply to — for [`/qe:copilot-review`](../skills/copilot-review/SKILL.md). Read-only: it never posts. Requires an authenticated `gh`; run `bash fetch-copilot.sh --help` for the argument forms. diff --git a/audit/scripts/README.md b/qe/scripts/audit/README.md similarity index 92% rename from audit/scripts/README.md rename to qe/scripts/audit/README.md index b744886..83997b2 100644 --- a/audit/scripts/README.md +++ b/qe/scripts/audit/README.md @@ -1,13 +1,13 @@ # Deterministic audit machinery -Fetching is mechanical, so it belongs here rather than in model judgement ([doctrine §4](../references/doctrine.md#4-checkpointing)). Stdlib only, driving `gh`; no install step. +Fetching is mechanical, so it belongs here rather than in model judgement ([doctrine §4](../../references/audit/doctrine.md#4-checkpointing)). Stdlib only, driving `gh`; no install step. ## `fetch_tracker.py` Snapshots a repository's whole tracker — issues and PRs, any state, with full comment threads — and reconciles the capture against the number sequence it should fill. ```bash -python ${CLAUDE_PLUGIN_ROOT}/scripts/fetch_tracker.py OWNER/REPO --out /snapshot +python ${CLAUDE_PLUGIN_ROOT}/scripts/audit/fetch_tracker.py OWNER/REPO --out /snapshot ``` | Output | Contents | diff --git a/audit/scripts/fetch_tracker.py b/qe/scripts/audit/fetch_tracker.py similarity index 100% rename from audit/scripts/fetch_tracker.py rename to qe/scripts/audit/fetch_tracker.py diff --git a/qe/scripts/benchmark/README.md b/qe/scripts/benchmark/README.md new file mode 100644 index 0000000..aea4e7c --- /dev/null +++ b/qe/scripts/benchmark/README.md @@ -0,0 +1,46 @@ +# The benchmark skill — scripts + +The deterministic core of `/qe:benchmark`: the shared scoring engine and the efficiency calibration. Developed and validated by [@xuanguang-li](https://github.com/xuanguang-li) on [lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654). + +## Layout + +``` +scoring/ + rubric.py the standard as code: evidence → score, deterministically + score.py engine/CLI: /evidence.json → results/scorecard.json + env_stamp.py provenance stamp: /results/env.json (+ failed steps) + EVIDENCE_TEMPLATE.json the judgement contract a new evaluation fills in +calibration/ + bellman_bench.py shared aiyagari Bellman benchmark — pins the "25× as-used + bellman_bench.json = score 5" efficiency anchor +``` + +All commands below run from the plugin root (`qe/` in this repo). + +The rubric in prose — dimensions, weights, anchors, checklists, verdict bands, worked HIGH/LOW examples — is [`../../references/benchmark/EVALUATION_FRAMEWORK.md`](../../references/benchmark/EVALUATION_FRAMEWORK.md). Two complete worked evaluations (measurement scripts, results, evidence, reports) live in [`../../references/benchmark/examples/`](../../references/benchmark/examples/) and double as the regression baseline the skill must reproduce. + +## How scoring works + +Scores are **never typed by hand** — each is a deterministic function of evidence: + +1. **Measure** — `python references/benchmark/examples//scripts/run_all.py` runs the per-lecture measurement scripts and writes `results/*.json` plus a provenance stamp (`results/env.json`: Python/platform/library versions and any failed steps — the seed of the QuantEcon/meta#335 shared result schema; generated per-run, not committed). The as-used steps repeat 3× per side in fresh processes; the headline speedup is a **median**, with per-run values kept for the contested-band check. +2. **Record evidence** — fill `/evidence.json` (copy `scoring/EVIDENCE_TEMPLATE.json`): measured numbers into the quantitative slots with their source, and each structural checklist item answered true/false **with a citation to the diff**. +3. **Score** — `python scripts/benchmark/scoring/score.py references/benchmark/examples/` applies `rubric.py` and writes `results/scorecard.json`, printing the derivation of every score, the final verdict (after the v2 correctness gates and the no-conversion rule), and the one-flip **sensitivity stamp** (robust/fragile with deciding flips). + +## Evaluating a new lecture + +Per-lecture measurement scripts are **adapted templates, not a fixed harness** — copy an existing example and adapt (this is the step the skill automates): + +```bash +conda activate quantecon # jax 0.4.x, numpy 2.x, quantecon +mkdir -p references/benchmark/examples//{scripts,results} +cp scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json references/benchmark/examples//evidence.json +# drop in model_old.py (from main) and model_new.py (from the PR branch), +# adapt check_equivalence / static_metrics / benchmark / as_used_total from an +# existing example, wire them into run_all.py, then: +python references/benchmark/examples//scripts/run_all.py +python scripts/benchmark/scoring/score.py references/benchmark/examples/ +# write _REPORT.md from the scorecard + evidence +``` + +Benchmarks are CPU-only; timings vary ±~15% run-to-run, so the rubric keys on orders of magnitude, not exact milliseconds. diff --git a/benchmark/scripts/calibration/bellman_bench.json b/qe/scripts/benchmark/calibration/bellman_bench.json similarity index 100% rename from benchmark/scripts/calibration/bellman_bench.json rename to qe/scripts/benchmark/calibration/bellman_bench.json diff --git a/benchmark/scripts/calibration/bellman_bench.py b/qe/scripts/benchmark/calibration/bellman_bench.py similarity index 100% rename from benchmark/scripts/calibration/bellman_bench.py rename to qe/scripts/benchmark/calibration/bellman_bench.py diff --git a/benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json b/qe/scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json similarity index 82% rename from benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json rename to qe/scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json index a5c8acf..28e97e5 100644 --- a/benchmark/scripts/scoring/EVIDENCE_TEMPLATE.json +++ b/qe/scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json @@ -1,7 +1,7 @@ { "lecture": "", "branch": "", - "_how": "Fill quantitative values from this lecture's results/*.json (note the source). Answer each structural criterion true/false with a citation to the diff. Then run: python scripts/scoring/score.py references/examples/ (from the plugin root). Never type a score — rubric.py computes it. score.py validates this file before scoring and refuses to run until the placeholders below are replaced: baseline_as_used_seconds is null here on purpose (it gates the no-conversion verdict, so it must be measured, not defaulted), and every criterion you set true needs an entry in that dimension's citations.", + "_how": "Fill quantitative values from this lecture's results/*.json (note the source). Answer each structural criterion true/false with a citation to the diff. Then run: python scripts/benchmark/scoring/score.py references/benchmark/examples/ (from the plugin root). Never type a score — rubric.py computes it. score.py validates this file before scoring and refuses to run until the placeholders below are replaced: baseline_as_used_seconds is null here on purpose (it gates the no-conversion verdict, so it must be measured, not defaulted), and every criterion you set true needs an entry in that dimension's citations.", "quantitative": { "correctness": { diff --git a/benchmark/scripts/scoring/env_stamp.py b/qe/scripts/benchmark/scoring/env_stamp.py similarity index 88% rename from benchmark/scripts/scoring/env_stamp.py rename to qe/scripts/benchmark/scoring/env_stamp.py index f886a21..cb1ebbd 100644 --- a/benchmark/scripts/scoring/env_stamp.py +++ b/qe/scripts/benchmark/scoring/env_stamp.py @@ -1,7 +1,7 @@ """Write /results/env.json — the provenance stamp for a measurement run. Usage: - python scripts/scoring/env_stamp.py [failed-step-title ...] + python scripts/benchmark/scoring/env_stamp.py [failed-step-title ...] Seed of the QuantEcon/meta#335 shared result + environment-descriptor schema: python/platform plus the versions of the measurement-relevant libraries. @@ -39,5 +39,5 @@ def main(lecture_dir, failed): if __name__ == "__main__": if len(sys.argv) < 2: - sys.exit("usage: python scripts/scoring/env_stamp.py [failed-step-title ...]") + sys.exit("usage: python scripts/benchmark/scoring/env_stamp.py [failed-step-title ...]") main(sys.argv[1], sys.argv[2:]) diff --git a/benchmark/scripts/scoring/rubric.py b/qe/scripts/benchmark/scoring/rubric.py similarity index 99% rename from benchmark/scripts/scoring/rubric.py rename to qe/scripts/benchmark/scoring/rubric.py index d3c7743..08debfb 100644 --- a/benchmark/scripts/scoring/rubric.py +++ b/qe/scripts/benchmark/scoring/rubric.py @@ -2,7 +2,7 @@ THE SCORING STANDARD (single source of truth, shared by every lecture). This module encodes the rubric described in prose in -../../references/EVALUATION_FRAMEWORK.md +../../../references/benchmark/EVALUATION_FRAMEWORK.md so that a score is a *deterministic function of evidence*, never a hand-typed number. Nothing here is lecture-specific: the same rubric is applied to every lecture. Only the per-lecture `evidence.json` changes. @@ -126,7 +126,7 @@ def score_correctness(builds, max_delta_shipped, matches_under_x64): `matches_under_x64` is the equivalence check re-run with JAX_ENABLE_X64=1: TRUE means the logic agrees once precision is removed from the question (residuals at x64 noise, ~1e-14 to ~1e-11, are recorded TRUE — see - references/examples/README.md). So FALSE asserts that the economics + references/benchmark/examples/README.md). So FALSE asserts that the economics genuinely differ, and it caps on its own. It deliberately does *not* require the shipped drift to be large as well: a candidate whose logic diverges but whose float32 output happens to agree closely is the exact diff --git a/benchmark/scripts/scoring/score.py b/qe/scripts/benchmark/scoring/score.py similarity index 95% rename from benchmark/scripts/scoring/score.py rename to qe/scripts/benchmark/scoring/score.py index 142f8a3..dda4add 100644 --- a/benchmark/scripts/scoring/score.py +++ b/qe/scripts/benchmark/scoring/score.py @@ -3,8 +3,8 @@ evidence and writes an auditable scorecard. Usage: - python scripts/scoring/score.py - # e.g. python scripts/scoring/score.py references/examples/ge_arrow + python scripts/benchmark/scoring/score.py + # e.g. python scripts/benchmark/scoring/score.py references/examples/ge_arrow It reads /evidence.json (inputs + citations; filled from results/) and writes /results/scorecard.json and prints the derivation table. @@ -156,7 +156,7 @@ def main(lecture_dir): "no_conversion": result["no_conversion"], "sensitivity": sens, "dimensions": result["rows"], - "_note": "Scores are computed by scripts/scoring/rubric.py from " + "_note": "Scores are computed by scripts/benchmark/scoring/rubric.py from " f"{lecture}/evidence.json; do not edit by hand.", } res_dir = os.path.join(lec_dir, "results") @@ -169,5 +169,5 @@ def main(lecture_dir): if __name__ == "__main__": if len(sys.argv) != 2: - sys.exit("usage: python scripts/scoring/score.py ") + sys.exit("usage: python scripts/benchmark/scoring/score.py ") main(sys.argv[1]) diff --git a/audit/skills/issues/SKILL.md b/qe/skills/audit-issues/SKILL.md similarity index 79% rename from audit/skills/issues/SKILL.md rename to qe/skills/audit-issues/SKILL.md index 55db1ea..5087ce4 100644 --- a/audit/skills/issues/SKILL.md +++ b/qe/skills/audit-issues/SKILL.md @@ -1,9 +1,9 @@ --- -name: issues +name: audit-issues description: "Audit every issue in a GitHub repository, open and closed — verify each status against the code rather than the thread, hunt fixed-but-open and never-landed-fix candidates, tier the open set into the repo's existing plan, and deliver a report bundle with a cross-link map. Read-only: it recommends tracker changes but never makes them. Use for a whole-tracker review, not a single issue." --- -# audit:issues +# qe:audit-issues Whole-tracker review of one repository. It works from a frozen snapshot and checkpoints as it goes. One measured run, on a 230-item tracker with 56 open issues: **22 minutes end to end**, of which phase 2 — verifying those 56 — was about 9, or roughly **10 seconds per open issue**. The two figures answer different questions: budget the total from the whole run, and scale it by the *open* count, since that is what phase 2 walks. Phase 2 also carries nearly all of the judgement, which is what makes a run expensive to re-derive rather than long to sit through. @@ -12,14 +12,14 @@ Whole-tracker review of one repository. It works from a frozen snapshot and chec ## Invocation ``` -/audit:issues [OWNER/REPO] [--out DIR] +/qe:audit-issues [OWNER/REPO] [--out DIR] ``` Both arguments are optional. With no repo, audit the current checkout's `origin`. Everything else is discovered: | Input | Discovery | On failure | |---|---|---| -| Project-notes system | The search order in [quantecon-context.md](../../references/quantecon-context.md#finding-the-plan-to-slot-into) | Say none was found; tier against milestones | +| Project-notes system | The search order in [quantecon-context.md](../../references/audit/quantecon-context.md#finding-the-plan-to-slot-into) | Say none was found; tier against milestones | | Label policy | [QEP-2](https://github.com/QuantEcon/qeps/pull/2), status read at run time | Treat as draft — recommend canonical labels only | | Work-plan anchor | The live plan or tracking issue in the tracker | Tier against the notes system alone | | Prior audits | Earlier audit bundles in the notes system; closed "priority order" issues | Note that this is the first audit | @@ -28,15 +28,15 @@ Ask only when discovery is ambiguous — two plausible plan anchors, say — not ## Read this first -- [doctrine.md](../../references/doctrine.md) — trust rules, evidence classes, the read-only boundary, phases, coverage self-audit -- [quantecon-context.md](../../references/quantecon-context.md) — repo types, label ownership, the cross-repo graph, the closing-keyword hazard -- [deliverables.md](../../references/deliverables.md) — what a report owes its reader, where it may land, and this skill's four-document shape +- [doctrine.md](../../references/audit/doctrine.md) — trust rules, evidence classes, the read-only boundary, phases, coverage self-audit +- [quantecon-context.md](../../references/audit/quantecon-context.md) — repo types, label ownership, the cross-repo graph, the closing-keyword hazard +- [deliverables.md](../../references/audit/deliverables.md) — what a report owes its reader, where it may land, and this skill's four-document shape Then, in the audited repo: the notes system, `CHANGELOG.md`, the latest release notes, and `AGENTS.md`/`CLAUDE.md`. ## Working directory -Runs happen in a checkout of the audited repo, because phase 2 verifies claims against its default branch. **Put the working directory somewhere the repo already ignores**, so a run leaves `git status` clean and no `.gitignore` edit is needed ([doctrine §3](../../references/doctrine.md#3-read-only-boundary)) — in that order: +Runs happen in a checkout of the audited repo, because phase 2 verifies claims against its default branch. **Put the working directory somewhere the repo already ignores**, so a run leaves `git status` clean and no `.gitignore` edit is needed ([doctrine §3](../../references/audit/doctrine.md#3-read-only-boundary)) — in that order: 1. An ignored scratch location in the notes system — `.dev/scratch/audit-/` in QuantEcon repos that have one, where `.dev/scratch/*` is already gitignored. 2. `.audit/-/` at the checkout root, left untracked and uncommitted. @@ -51,14 +51,14 @@ State which was used in the report's method section. Everything the run produces | `links.md` | phase 3 | the cross-link graph | | `01-…` `02-…` `03-…` `README.md` | phase 4 | the delivered bundle | -**Append to the checkpoint as you go, not when the phase ends** ([doctrine §4](../../references/doctrine.md#4-checkpointing)). Phase 2 carries nearly all the judgement, so it is both the phase an interruption lands inside rather than between, and the phase whose log the catalog is later assembled from. +**Append to the checkpoint as you go, not when the phase ends** ([doctrine §4](../../references/audit/doctrine.md#4-checkpointing)). Phase 2 carries nearly all the judgement, so it is both the phase an interruption lands inside rather than between, and the phase whose log the catalog is later assembled from. **Both passes are checkpointed, and the resume rule reads `issues.json`.** Phase 2 walks two sets — the open issues and the closed ones — and each has its own section in `findings.md`. On restart, partition `issues.json` by state and, for each partition independently, resume at the lowest number with no entry under the matching heading; re-verify the last entry in each rather than trusting a possibly truncated write. Do not infer progress from the file's length or from a single block: run 1 wrote only the open set to `findings.md` and sent the 62 closed issues straight to the catalog, so a resume would have re-verified all 62 from scratch while reporting itself complete ([defect 2](https://github.com/QuantEcon/skills/issues/21)). ## Phase 1 — snapshot ```bash -python ${CLAUDE_PLUGIN_ROOT}/scripts/fetch_tracker.py OWNER/REPO --out /snapshot +python ${CLAUDE_PLUGIN_ROOT}/scripts/audit/fetch_tracker.py OWNER/REPO --out /snapshot ``` Preflight fails in the first minute on missing `gh`, missing auth, or an invisible repo — read the error rather than working around it. The snapshot captures issues and PRs in any state with full comment threads, so closed threads cost nothing extra to read and there is no excuse for skipping them. @@ -69,7 +69,7 @@ Read `coverage.json` before proceeding. A stream at the fetch limit, or unaccoun ## Phase 2 — verify -Per [doctrine §1](../../references/doctrine.md#1-what-makes-a-bulk-audit-trustworthy), the thread is a hypothesis and the default branch is the evidence. +Per [doctrine §1](../../references/audit/doctrine.md#1-what-makes-a-bulk-audit-trustworthy), the thread is a hypothesis and the default branch is the evidence. **Each open issue.** Does the complaint still reproduce on the default branch — check the named file or function, not the claim about it? Did a merged PR or a campaign claim it, and does the diff actually contain the change (this is where wave escapes hide)? Is it a duplicate of another open issue — same fix, no mutual link? Is it superseded, where a newer issue is the better carrier — then port the lessons across and close the old one as superseded, never silently. Is it blocked on an external decision — annotate the hold and split live from gated scope, so the issue does not read as wholly frozen when part of it is actionable. For parents and trackers: are the children attached natively, and is it still the plan of record? @@ -79,7 +79,7 @@ Per [doctrine §1](../../references/doctrine.md#1-what-makes-a-bulk-audit-trustw Sibling-repo checks belong here too: for QuantEcon, "resolved in a sibling" and "one step of a rollout" are the two most common wrong conclusions a single-repo audit reaches. -**Check every citation against the baseline ref before tagging it `[verified]`** ([doctrine §2](../../references/doctrine.md#2-evidence-classes)): +**Check every citation against the baseline ref before tagging it `[verified]`** ([doctrine §2](../../references/audit/doctrine.md#2-evidence-classes)): - **a commit** — run `git merge-base --is-ancestor `; exit 0 means citable - **a `file:line`** — confirm that line says what the finding claims *on the ref*, not in the working tree @@ -87,7 +87,7 @@ Sibling-repo checks belong here too: for QuantEcon, "resolved in a sibling" and Evidence that lives only on an unmerged branch is still worth citing — as the open PR it is, tagged `[stated]` or `[inferred]`. -Write each item's finding to `findings.md` as it is verified — the closed pass too, under its own heading, not straight into the catalog — in the catalog entry format from [deliverables.md](../../references/deliverables.md#the-auditissues-bundle), so phase 4 assembles the catalog rather than re-deriving it and an interrupted run loses one item rather than the phase. +Write each item's finding to `findings.md` as it is verified — the closed pass too, under its own heading, not straight into the catalog — in the catalog entry format from [deliverables.md](../../references/audit/deliverables.md#the-qeaudit-issues-bundle), so phase 4 assembles the catalog rather than re-deriving it and an interrupted run loses one item rather than the phase. ## Phase 3 — relate @@ -107,14 +107,14 @@ Slot into the repo's existing plan; never invent a parallel one. Tier by repo ty | **T2** | Production quality, grouped by family — issues sharing a fix-shape are one unit of work. | | **T3** | Decision-gated, external-dependency, deliberately deferred, parking lots. | -Priority labels only for genuine outliers, a handful either way, and only if the policy provides them. Then write the bundle per [deliverables.md](../../references/deliverables.md), as `01-issue-triage-report.md`, `02-issue-catalog.md`, `03-issue-links.md`, `README.md`. +Priority labels only for genuine outliers, a handful either way, and only if the policy provides them. Then write the bundle per [deliverables.md](../../references/audit/deliverables.md), as `01-issue-triage-report.md`, `02-issue-catalog.md`, `03-issue-links.md`, `README.md`. **Scale the bundle to the tracker.** Four documents suit a tracker big enough that the argument, the enumeration and the graph get in each other's way. Below roughly 30 open issues they do not: fold the catalog and the link graph into the report, keep the `README.md` index, and say in the coverage statement which shape was used. Padding a small audit into four files makes it *less* checkable, which is the one thing the shape exists to protect. ## Phase 5 — self-audit -Run [doctrine §5](../../references/doctrine.md#5-coverage-self-audit) against `coverage.json`: reconcile the counts, explain every unaccounted number, confirm threads were read on both the open and closed sides, and state the residue — inline review comments, GraphQL-only data, anything after the snapshot timestamp. Fold any change back into the documents rather than appending a correction. +Run [doctrine §5](../../references/audit/doctrine.md#5-coverage-self-audit) against `coverage.json`: reconcile the counts, explain every unaccounted number, confirm threads were read on both the open and closed sides, and state the residue — inline review comments, GraphQL-only data, anything after the snapshot timestamp. Fold any change back into the documents rather than appending a correction. ## What this skill does not do -It does not close, label, comment, or edit — see [doctrine §3](../../references/doctrine.md#3-read-only-boundary). The tracker pass is a separate, human-driven step working from the bundle, and label application belongs to `qe gh labels`. Judging one PR's technical quality is likewise out of scope; that is a per-item review, not a portfolio audit. +It does not close, label, comment, or edit — see [doctrine §3](../../references/audit/doctrine.md#3-read-only-boundary). The tracker pass is a separate, human-driven step working from the bundle, and label application belongs to `qe gh labels`. Judging one PR's technical quality is likewise out of scope; that is a per-item review, not a portfolio audit. diff --git a/benchmark/skills/review-acceleration/SKILL.md b/qe/skills/benchmark/SKILL.md similarity index 60% rename from benchmark/skills/review-acceleration/SKILL.md rename to qe/skills/benchmark/SKILL.md index e7a6482..515b9d6 100644 --- a/benchmark/skills/review-acceleration/SKILL.md +++ b/qe/skills/benchmark/SKILL.md @@ -1,11 +1,11 @@ --- -name: review-acceleration +name: benchmark description: Advise whether a QuantEcon lecture is worth converting to an accelerated implementation (JAX, Numba), or review whether an existing conversion PR actually improves the lecture. Triage mode needs only the lecture — it measures the as-used baseline, bounds the possible win, and recommends convert or don't-convert with the binding constraint named. Review mode measures a candidate against the baseline (as-used performance in a fresh process with JIT compile time included, numerical fidelity under float32/float64, readability) and produces a scored report with a merge recommendation. Use when deciding whether to convert a lecture to JAX, or when reviewing a NumPy→JAX/Numba conversion PR. --- -# review-acceleration +# benchmark -> **Status: evaluation system landed (rubric v2); skill wired for workspace runs; triage-first since 0.4.0.** The system was developed and validated by @xuanguang-li on [QuantEcon/lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) and now lives in this plugin: the rubric in [`references/EVALUATION_FRAMEWORK.md`](../../references/EVALUATION_FRAMEWORK.md), the deterministic scoring engine in `scripts/scoring/`, and two complete worked evaluations in `references/examples/`. Rubric v2 (verdict gates, no-conversion, sensitivity stamp, K-repeat as-used) implements the surviving critiques of the 2026-07-21 design review. Tracking: [QuantEcon/skills#4](https://github.com/QuantEcon/skills/issues/4), [QuantEcon/meta#335](https://github.com/QuantEcon/meta/issues/335) (workstream B). +> **Status: evaluation system landed (rubric v2); skill wired for workspace runs; triage-first since benchmark 0.4.0 (2026-08-07).** The system was developed and validated by @xuanguang-li on [QuantEcon/lecture-python.myst#717](https://github.com/QuantEcon/lecture-python.myst/pull/717) and [#654](https://github.com/QuantEcon/lecture-python.myst/pull/654) and now lives in the `qe` plugin: the rubric in [`references/benchmark/EVALUATION_FRAMEWORK.md`](../../references/benchmark/EVALUATION_FRAMEWORK.md), the deterministic scoring engine in `scripts/benchmark/scoring/`, and two complete worked evaluations in `references/benchmark/examples/`. Rubric v2 (verdict gates, no-conversion, sensitivity stamp, K-repeat as-used) implements the surviving critiques of the 2026-07-21 design review. Tracking: [QuantEcon/skills#4](https://github.com/QuantEcon/skills/issues/4), [QuantEcon/meta#335](https://github.com/QuantEcon/meta/issues/335) (workstream B). ## Which mode @@ -22,17 +22,17 @@ QuantEcon lectures are teaching materials first and programs second. A rewrite t ## Where things live at run time -When this skill runs from the installed plugin, the plugin's files are **read-only** at `${CLAUDE_PLUGIN_ROOT}` (the engine in `${CLAUDE_PLUGIN_ROOT}/scripts/scoring/`, templates and worked examples in `${CLAUDE_PLUGIN_ROOT}/references/examples/`). The evaluation itself is built in the **user's workspace** — normally the lecture repo checkout under review: +When this skill runs from the installed plugin, the plugin's files are **read-only** at `${CLAUDE_PLUGIN_ROOT}` (the engine in `${CLAUDE_PLUGIN_ROOT}/scripts/benchmark/scoring/`, templates and worked examples in `${CLAUDE_PLUGIN_ROOT}/references/benchmark/examples/`). The evaluation itself is built in the **user's workspace** — normally the lecture repo checkout under review: ``` /benchmark-eval// scripts/ # adapted per-lecture from a worked example's scripts/ - evidence.json # started from ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/EVIDENCE_TEMPLATE.json + evidence.json # started from ${CLAUDE_PLUGIN_ROOT}/scripts/benchmark/scoring/EVIDENCE_TEMPLATE.json results/ # written by the pipeline + scorer _REPORT.md ``` -Never write into the plugin directory. Scoring works on any directory: `python ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/score.py benchmark-eval/`. The scaffolded `run_all.py` reads `CLAUDE_PLUGIN_ROOT` from the environment to find the shared engine — export it (or keep the adapted script's path pointing at the plugin) before running the pipeline. +Never write into the plugin directory. Scoring works on any directory: `python ${CLAUDE_PLUGIN_ROOT}/scripts/benchmark/scoring/score.py benchmark-eval/`. The scaffolded `run_all.py` reads `CLAUDE_PLUGIN_ROOT` from the environment to find the shared engine — export it (or keep the adapted script's path pointing at the plugin) before running the pipeline. **Preconditions to verify before starting** (fail loudly, don't improvise silently): a checkout of the lecture repo — for review mode, with both refs fetchable (baseline, usually `main`, and the candidate branch); a Python environment with `jax`, `numpy`, and the lecture's imports (the reference runs used the `quantecon` conda env); CPU-only is the calibrated regime. Record the environment via the provenance stamp — `run_all.py` does this automatically, including failed-step titles, so a partial run cannot claim full provenance. @@ -47,26 +47,26 @@ The decision criteria themselves are canonical in the manual's JAX style page 3. **Crossover check**: the lecture's problem sizes vs warm crossover-n. 4. **Readability-cost forecast**: which prerequisite concepts the conversion would force. -Decision rule ([framework §1](../../references/EVALUATION_FRAMEWORK.md)): the rubric weights readability above efficiency on purpose, so a conversion that costs meaningful readability cannot break even on speed alone — the weight algebra behind that is the framework's, not restated here — and structural wins are usually achievable in the baseline library. Report a recommendation — **convert or don't-convert** — with the binding constraint named, not a scorecard. Validated 2026-07-21: blind triage on ge_arrow, markov_asset (both sub-second baselines → don't convert) and the aiyagari pattern (~54 s → convert) reproduced all three known verdicts, from the triage-time baseline measurements recorded in the plugin README; triage cannot predict conversion-quality defects (markov_asset's build bug), and must say so. Rubric v2 closes the loop from the review side: when a full evaluation's efficiency evidence shows the don't-convert profile (baseline under the 1 s floor, candidate slower as-used), the scorecard itself emits the **no-conversion** verdict — review and triage can no longer disagree on that question. +Decision rule ([framework §1](../../references/benchmark/EVALUATION_FRAMEWORK.md)): the rubric weights readability above efficiency on purpose, so a conversion that costs meaningful readability cannot break even on speed alone — the weight algebra behind that is the framework's, not restated here — and structural wins are usually achievable in the baseline library. Report a recommendation — **convert or don't-convert** — with the binding constraint named, not a scorecard. Validated 2026-07-21: blind triage on ge_arrow, markov_asset (both sub-second baselines → don't convert) and the aiyagari pattern (~54 s → convert) reproduced all three known verdicts, from the triage-time baseline measurements recorded in the plugin README; triage cannot predict conversion-quality defects (markov_asset's build bug), and must say so. Rubric v2 closes the loop from the review side: when a full evaluation's efficiency evidence shows the don't-convert profile (baseline under the 1 s floor, candidate slower as-used), the scorecard itself emits the **no-conversion** verdict — review and triage can no longer disagree on that question. ## Review — did this conversion improve the lecture? Given a baseline implementation (usually `main`) and a candidate (usually a PR branch) for one lecture, follow the measure → record-evidence → score contract in [`scripts/README.md`](../../scripts/README.md) — **scores are never typed by hand**: -1. **Scaffold** — create `/benchmark-eval//` from a worked example under `${CLAUDE_PLUGIN_ROOT}/references/examples/`: extract `model_old.py` (baseline) and `model_new.py` (candidate) **verbatim** from the lecture's code cells (disclose any deviation in the report), and adapt the measurement templates (`check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, plus lecture-specific ones) to the lecture's actual examples and call sequence. Adapting templates per lecture is this skill's job — there is deliberately no rigid harness. Before measuring, diff the extracted code and the replayed call sequence against the lecture's cells and fix mismatches — construction-pattern drift here invalidates everything downstream. +1. **Scaffold** — create `/benchmark-eval//` from a worked example under `${CLAUDE_PLUGIN_ROOT}/references/benchmark/examples/`: extract `model_old.py` (baseline) and `model_new.py` (candidate) **verbatim** from the lecture's code cells (disclose any deviation in the report), and adapt the measurement templates (`check_equivalence.py`, `static_metrics.py`, `benchmark.py`, `as_used_total.py`, plus lecture-specific ones) to the lecture's actual examples and call sequence. Adapting templates per lecture is this skill's job — there is deliberately no rigid harness. Before measuring, diff the extracted code and the replayed call sequence against the lecture's cells and fix mismatches — construction-pattern drift here invalidates everything downstream. 2. **Measure** — `run_all.py`: equivalence under the default dtype AND `jax_enable_x64` (report `max|Δ|` per regime); static metrics (prerequisite concepts, docstring coverage); the **as-used benchmark** — replay the lecture's *actual* solver call sequence at its *actual* sizes in a fresh interpreter so trace/compile time counts, repeated ≥3 times per side with the **median** as the headline (`as_used_speedup = baseline median / candidate median`), with warm timings alongside (never alone), a crossover-n scaling curve, and a recompile audit. A provenance stamp (`results/env.json`, generated per-run) records the environment and any failed steps. 3. **Record evidence** — fill `evidence.json` from the results: measured numbers into the quantitative slots with sources (including `baseline_as_used_seconds` and the per-run `as_used_runs`); each structural checklist item answered true/false **with a citation to the diff**. This file is the only place judgement is recorded. -4. **Score** — `python ${CLAUDE_PLUGIN_ROOT}/scripts/scoring/score.py benchmark-eval/` computes all seven dimensions and the weighted total deterministically. The weights, threshold anchors, and verdict bands are defined in [`references/EVALUATION_FRAMEWORK.md`](../../references/EVALUATION_FRAMEWORK.md) §1–2 and machine-encoded in `scripts/scoring/rubric.py` — never restate or re-derive them here. v2 outputs you must carry into the report verbatim: the **verdict gate** (correctness 1/2 caps the band), the **no-conversion** verdict (don't-convert profile beats polish), and the **sensitivity stamp** (robust / robust-at-floor / fragile, with the deciding flips). Carry the stamp as printed — *robust-at-floor* means the outcome held only because it is already in the bottom band and could not get worse, so never report it as *robust*. Because one stamp currently covers both measurement and judgement perturbations ([framework §1](../../references/EVALUATION_FRAMEWORK.md)), quote the deciding-flip list rather than resting the report's confidence on the word alone. +4. **Score** — `python ${CLAUDE_PLUGIN_ROOT}/scripts/benchmark/scoring/score.py benchmark-eval/` computes all seven dimensions and the weighted total deterministically. The weights, threshold anchors, and verdict bands are defined in [`references/benchmark/EVALUATION_FRAMEWORK.md`](../../references/benchmark/EVALUATION_FRAMEWORK.md) §1–2 and machine-encoded in `scripts/benchmark/scoring/rubric.py` — never restate or re-derive them here. v2 outputs you must carry into the report verbatim: the **verdict gate** (correctness 1/2 caps the band), the **no-conversion** verdict (don't-convert profile beats polish), and the **sensitivity stamp** (robust / robust-at-floor / fragile, with the deciding flips). Carry the stamp as printed — *robust-at-floor* means the outcome held only because it is already in the bottom band and could not get worse, so never report it as *robust*. Because one stamp currently covers both measurement and judgement perturbations ([framework §1](../../references/benchmark/EVALUATION_FRAMEWORK.md)), quote the deciding-flip list rather than resting the report's confidence on the word alone. 5. **Report** — write `_REPORT.md` from the scorecard + evidence, following the worked examples' format and **leading with the decision**: the TL;DR opens with the *full* verdict (including gate/no-conversion/sensitivity) and carries the weighted score alongside as candidate quality for the record. Then the dimension table with drivers — including a verdict row, so the table still carries the decision when it is quoted on its own — evidence per dimension, and a must-fix list mapping each recommendation to the dimension it lifts. The scorecard's two outputs are different layers: the total measures the candidate's polish, the verdict carries the recommendation, and a report must never present the number where the decision belongs (skills#14, finding 6: careful readers took the total for the headline twice). Never present warm-only speedups as the headline — the ge_arrow case measured 1.4–4.8× faster warm and ~45× slower as-used. ## Calibration baseline (regression anchors) -The two worked evaluations in `references/examples/` are the validation baseline — re-running their pipelines must reproduce these verdicts. Confirmed end-to-end 2026-07-22: a fresh-checkout workspace run of ge_arrow (#717, base `8cfba4c`) on a different machine and jax **0.10.1** (reference: 0.4.35) reproduced 2.85 / no-conversion / fragile with the same deciding flips — every measured quantity moved only within its band. Evidence files record `source_pr` + base/head SHAs: +The two worked evaluations in `references/benchmark/examples/` are the validation baseline — re-running their pipelines must reproduce these verdicts. Confirmed end-to-end 2026-07-22: a fresh-checkout workspace run of ge_arrow (#717, base `8cfba4c`) on a different machine and jax **0.10.1** (reference: 0.4.35) reproduced 2.85 / no-conversion / fragile with the same deciding flips — every measured quantity moved only within its band. Evidence files record `source_pr` + base/head SHAs: - **`ge_arrow`** ([#717](https://github.com/QuantEcon/lecture-python.myst/pull/717)): **2.85/5 — no-conversion** (candidate band mixed/wash; sensitivity: fragile). Tiny 2×2/3×3 economies, fresh static args per call → ~45× slower as-used despite warm wins, on a 0.035 s baseline. - **`markov_asset`** ([#654](https://github.com/QuantEcon/lecture-python.myst/pull/654)): **2.25/5 — no-conversion + gated net regression** (sensitivity: robust-at-floor). A stray `err.throw()` that crashes in any clean namespace and, in notebook order, silently disables the checkify stability validation (a masked failure — see the REPORT erratum); float32 drift near a critical stability margin. -- **HIGH anchor:** the aiyagari Bellman pattern (`scripts/calibration/bellman_bench.py`) — large fixed-shape arrays, many re-solves; ~25× faster as-used → the "score 5" calibration. +- **HIGH anchor:** the aiyagari Bellman pattern (`scripts/benchmark/calibration/bellman_bench.py`) — large fixed-shape arrays, many re-solves; ~25× faster as-used → the "score 5" calibration. The rubric will also be distilled into the QuantEcon manual as the companion to the JAX style page ([QuantEcon.manual#104](https://github.com/QuantEcon/QuantEcon.manual/issues/104)). diff --git a/qe/skills/workplan/SKILL.md b/qe/skills/workplan/SKILL.md index 8fd4b32..7eb0deb 100644 --- a/qe/skills/workplan/SKILL.md +++ b/qe/skills/workplan/SKILL.md @@ -71,7 +71,7 @@ Shared machinery for `read`, `resume`, `update`, and `create`'s session-bootstra ## `create` — open a plan -**1. Gather — three sources.** *Backlog triage*: sweep the repo's open issues, PRs and recent activity, and organise the *agreed* work — a planning pass, not an audit (for a whole-tracker review that re-verifies every issue against the code, run [`/audit:issues`](https://github.com/QuantEcon/skills/blob/main/audit/skills/issues/SKILL.md) first and feed its report to `workplan-project`). *Session bootstrap*: the session did work with no plan open — anchor-and-sweep over the session's traces plus the stated goals for next session; the first plan is a handover with no predecessor. *Succession*: the carry-forward register handed over by `close` — the register, never the old body, is the input; add what the closing session surfaced. When the source isn't obvious from how the verb was reached, ask. +**1. Gather — three sources.** *Backlog triage*: sweep the repo's open issues, PRs and recent activity, and organise the *agreed* work — a planning pass, not an audit (for a whole-tracker review that re-verifies every issue against the code, run [`/qe:audit-issues`](https://github.com/QuantEcon/skills/blob/main/qe/skills/audit-issues/SKILL.md) first and feed its report to `workplan-project`). *Session bootstrap*: the session did work with no plan open — anchor-and-sweep over the session's traces plus the stated goals for next session; the first plan is a handover with no predecessor. *Succession*: the carry-forward register handed over by `close` — the register, never the old body, is the input; add what the closing session surfaced. When the source isn't obvious from how the verb was reached, ask. **2. Verify the baseline.** Every fact the plan will rest on is measured *now* and stamped: tag positions, coverage counts, open-PR sets, CI state — whatever the work blocks depend on. Each row of the live-state table says what was measured and when, because a future session will trust it without seeing how it was built. What cannot be verified now goes in as an open question, not a fact. diff --git a/reviews/audit-run-action-translation-2026-07-28.md b/reviews/audit-run-action-translation-2026-07-28.md index d9582a4..4c60211 100644 --- a/reviews/audit-run-action-translation-2026-07-28.md +++ b/reviews/audit-run-action-translation-2026-07-28.md @@ -28,9 +28,9 @@ Unlike the [ge_arrow validation run](validation-run-ge_arrow-2026-07-22.md), the | 4 — write (3 documents + index) | 17:08:05 | ~10 min | | **Total** | | **~22 min** for 230 items | -[`audit/README.md`](../audit/README.md) says *"Expect hours, not minutes, on a repo with a hundred items"*; [`SKILL.md`](../audit/skills/issues/SKILL.md) says *"a hundred-issue repo is a multi-hour run"*. Both are wrong by roughly an order of magnitude on this repo. +[`audit/README.md`](https://github.com/QuantEcon/skills/blob/audit--v0.2.0/audit/README.md) says *"Expect hours, not minutes, on a repo with a hundred items"*; [`SKILL.md`](https://github.com/QuantEcon/skills/blob/audit--v0.2.0/audit/skills/issues/SKILL.md) says *"a hundred-issue repo is a multi-hour run"*. Both are wrong by roughly an order of magnitude on this repo. -This is uncomfortable rather than merely inaccurate, because it partly undercuts its own machinery. [doctrine §4](../audit/references/doctrine.md#4-checkpointing) justifies phase checkpointing on the premise that *"bulk audits outlive sessions"*. At 22 minutes they do not. Checkpointing is not thereby worthless — a 1000-item tracker scales up, interruption remains possible, and the incremental log turned out to be independently useful as an audit trail — but the stated *reason* is false and should be corrected rather than quietly retained. See defect 4. +This is uncomfortable rather than merely inaccurate, because it partly undercuts its own machinery. [doctrine §4](https://github.com/QuantEcon/skills/blob/audit--v0.2.0/audit/references/doctrine.md#4-checkpointing) justifies phase checkpointing on the premise that *"bulk audits outlive sessions"*. At 22 minutes they do not. Checkpointing is not thereby worthless — a 1000-item tracker scales up, interruption remains possible, and the incremental log turned out to be independently useful as an audit trail — but the stated *reason* is false and should be corrected rather than quietly retained. See defect 4. ## The claims under test @@ -55,11 +55,11 @@ Defect 2 makes this worse than a neutral gap: the resume rule as written would h ## Defects found -1. **A `[verified]` citation that does not verify, on the headline finding.** The #91 wave-escape entry cites `b99b431` as where `docs/user/heading-maps.md`'s history ends. That commit is real and does touch the file — but it is **not on `main`**; it exists only on `origin/fix/heading-map-position-fallback`, unmerged, dated 2026-03-24. The last commit touching that path on `main` is `0ea2539`. The audit's own header says "verified against `main` @ `2c3d624`". The conclusion survives independent checking and is arguably strengthened, but a reviewer who checks the citation finds nothing — worse than no citation. [doctrine §2](../audit/references/doctrine.md#2-evidence-classes) says cite `file:line`, a merged PR, or a tag; it never says *confirm the commit is an ancestor of the ref you named*. One `git merge-base --is-ancestor` closes it. +1. **A `[verified]` citation that does not verify, on the headline finding.** The #91 wave-escape entry cites `b99b431` as where `docs/user/heading-maps.md`'s history ends. That commit is real and does touch the file — but it is **not on `main`**; it exists only on `origin/fix/heading-map-position-fallback`, unmerged, dated 2026-03-24. The last commit touching that path on `main` is `0ea2539`. The audit's own header says "verified against `main` @ `2c3d624`". The conclusion survives independent checking and is arguably strengthened, but a reviewer who checks the citation finds nothing — worse than no citation. [doctrine §2](https://github.com/QuantEcon/skills/blob/audit--v0.2.0/audit/references/doctrine.md#2-evidence-classes) says cite `file:line`, a merged PR, or a tag; it never says *confirm the commit is an ancestor of the ref you named*. One `git merge-base --is-ancestor` closes it. 2. **The closed side was never checkpointed.** `findings.md` contains one section — `## Open issues`, 56 entries. The 62 closed issues were verified and appear in the catalog, but went straight to the deliverable. Half the phase-2 work sat outside the checkpoint, and it breaks the resume rule from #17: *"resume at the lowest number in `issues.json` with no entry"* would have re-verified all 62 closed issues from scratch. A defect in the fix, found by running it. -3. **The bundle landed where it cannot be delivered.** [`deliverables.md`](../audit/references/deliverables.md) names `.dev/audits/-/` as the destination; the bundle is in `.dev/scratch/`, which is gitignored — so it cannot be committed where it sits. The run correctly followed SKILL.md's working-directory table, which lists the bundle under `--out`. SKILL.md and deliverables.md now contradict each other, introduced by #17. The convention side is now a suggestion at [QuantEcon.manual#140](https://github.com/QuantEcon/QuantEcon.manual/issues/140). +3. **The bundle landed where it cannot be delivered.** [`deliverables.md`](https://github.com/QuantEcon/skills/blob/audit--v0.2.0/audit/references/deliverables.md) names `.dev/audits/-/` as the destination; the bundle is in `.dev/scratch/`, which is gitignored — so it cannot be committed where it sits. The run correctly followed SKILL.md's working-directory table, which lists the bundle under `--out`. SKILL.md and deliverables.md now contradict each other, introduced by #17. The convention side is now a suggestion at [QuantEcon.manual#140](https://github.com/QuantEcon/QuantEcon.manual/issues/140). 4. **The cost claim is wrong** (see above), and with it doctrine §4's stated justification for checkpointing. diff --git a/reviews/benchmark-design-2026-07-21-independent.md b/reviews/benchmark-design-2026-07-21-independent.md index 66822ea..13cb672 100644 --- a/reviews/benchmark-design-2026-07-21-independent.md +++ b/reviews/benchmark-design-2026-07-21-independent.md @@ -3,7 +3,7 @@ **Date:** 2026-07-21 **Branch reviewed:** `docs-skills-usage` **Scope:** the full evaluation system in `benchmark/` — rubric dimensions and weights, metrics, aggregation and verdict method, thresholds and calibration, measurement architecture. Everything treated as open to revision; the goal is the best possible evaluation system for QuantEcon, where lectures are teaching materials first. -**Method:** read every design document ([README.md](../benchmark/README.md), [EVALUATION_FRAMEWORK.md](../benchmark/references/EVALUATION_FRAMEWORK.md), [scripts/README.md](../benchmark/scripts/README.md), [SKILL.md](../benchmark/skills/review-acceleration/SKILL.md), [examples README](../benchmark/references/examples/README.md)), the scoring engine ([rubric.py](../benchmark/scripts/scoring/rubric.py), [score.py](../benchmark/scripts/scoring/score.py)), both worked evaluations (evidence, results, reports), and the measurement scripts; then ran the rubric against synthetic edge cases to test the aggregation empirically. The numbered edge-case results below (A–D) were produced by executing `rubric.score_all` directly on constructed evidence. +**Method:** read every design document ([README.md](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/README.md), [EVALUATION_FRAMEWORK.md](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/references/EVALUATION_FRAMEWORK.md), [scripts/README.md](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/scripts/README.md), [SKILL.md](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/skills/review-acceleration/SKILL.md), [examples README](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/references/examples/README.md)), the scoring engine ([rubric.py](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/scripts/scoring/rubric.py), [score.py](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/scripts/scoring/score.py)), both worked evaluations (evidence, results, reports), and the measurement scripts; then ran the rubric against synthetic edge cases to test the aggregation empirically. The numbered edge-case results below (A–D) were produced by executing `rubric.score_all` directly on constructed evidence. ## Bottom line diff --git a/reviews/triage-run-wald_friedman-2026-07-27.md b/reviews/triage-run-wald_friedman-2026-07-27.md index 427b6be..6cb33fd 100644 --- a/reviews/triage-run-wald_friedman-2026-07-27.md +++ b/reviews/triage-run-wald_friedman-2026-07-27.md @@ -1,6 +1,6 @@ # Triage run — wald_friedman evaluated with no candidate (2026-07-27) -The first end-to-end exercise of `/benchmark:review-acceleration` in **triage mode** — the prospective, no-candidate path described in [`SKILL.md` § "Triage mode (no candidate yet)"](../benchmark/skills/review-acceleration/SKILL.md) — against a lecture that had never been evaluated. Two purposes: answer the conversion question for a real Table C candidate, and test the path that the two committed worked examples (both review-mode) do not cover. The plugin findings it produced are filed as [skills#14](https://github.com/QuantEcon/skills/issues/14); the lecture defect it tripped over is [lecture-python.myst#1008](https://github.com/QuantEcon/lecture-python.myst/issues/1008). Nothing was posted to any upstream PR. +The first end-to-end exercise of `/benchmark:review-acceleration` in **triage mode** — the prospective, no-candidate path described in [`SKILL.md` § "Triage mode (no candidate yet)"](https://github.com/QuantEcon/skills/blob/benchmark--v0.4.0/benchmark/skills/review-acceleration/SKILL.md) — against a lecture that had never been evaluated. Two purposes: answer the conversion question for a real Table C candidate, and test the path that the two committed worked examples (both review-mode) do not cover. The plugin findings it produced are filed as [skills#14](https://github.com/QuantEcon/skills/issues/14); the lecture defect it tripped over is [lecture-python.myst#1008](https://github.com/QuantEcon/lecture-python.myst/issues/1008). Nothing was posted to any upstream PR. Target chosen from the July 2026 rng-triage / JAX-candidacy review, whose Table C tags four lectures **Benchmark** — `navy_captain`, `wald_friedman`, `wald_friedman_2`, `likelihood_ratio_process_2` — none of which has a PR open. That absence is what forces triage mode: there is no candidate implementation to measure against.