From db5bbc6b7996f58100e54d27cfffe770872ff5a7 Mon Sep 17 00:00:00 2001 From: Operator Date: Mon, 31 Aug 2026 16:51:31 +0000 Subject: [PATCH 1/5] fix: enhancement eval history observability with repo sha provenance and judge drift markers Resolves #4 https://github.com/coston/executant/issues/4 --- ARCHITECTURE.md | 14 +- BACKLOG.md | 12 ++ README.md | 2 +- docs/eval-comparison.md | 85 +++++++++- package.json | 6 +- src/eval/export.ts | 40 ++++- src/eval/history.ts | 143 +++++++++++++++++ src/eval/index.ts | 32 +++- src/eval/provenance.ts | 115 ++++++++++++++ src/eval/report.ts | 39 +++++ src/eval/runner.ts | 15 +- src/eval/trend-index.ts | 94 +++++++++++ src/eval/types.ts | 26 ++++ src/lib/statusline.ts | 33 ++++ src/tasks/command.ts | 28 +++- src/tasks/stream.ts | 7 + src/tests/command.test.ts | 35 +++++ src/tests/eval-comparison.test.ts | 59 ++++++- src/tests/eval-history.test.ts | 250 ++++++++++++++++++++++++++++++ src/tests/eval-provenance.test.ts | 167 ++++++++++++++++++++ src/tests/eval-trend.test.ts | 102 ++++++++++++ src/tests/eval.test.ts | 42 ++--- src/tests/statusline-ui.test.ts | 10 +- src/tests/statusline.test.ts | 46 ++++++ src/ui/App.tsx | 6 +- src/ui/StatusBar.tsx | 18 ++- 26 files changed, 1379 insertions(+), 47 deletions(-) create mode 100644 src/eval/history.ts create mode 100644 src/eval/provenance.ts create mode 100644 src/eval/trend-index.ts create mode 100644 src/tests/eval-history.test.ts create mode 100644 src/tests/eval-provenance.test.ts create mode 100644 src/tests/eval-trend.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6a778f2..fee7766 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -159,19 +159,25 @@ Large text passed to Claude lives in `src/prompts/*.txt`. They use `{{PLACEHOLDE The eval system tests and iteratively refines the prompt templates in `src/prompts/`. It is not user-facing — run via `npm run eval` during development. -**`src/eval/index.ts`** — CLI entry point. Parses `--refine`, `--max-iter`, `--models`, `--cases`, `--output-json`, and `--output-csv` flags. Accepts one or more eval file paths as positional arguments. `--cases` accepts comma-separated case IDs or 1-based index ranges (e.g. `simple,1-3`) to run a subset without editing YAML. Single-model mode: loads existing CSV results for resume (skips already-scored cases), runs remaining cases, optional refine loop. Multi-model mode (2+ models via `--models`): runs each model independently, builds an `EvalComparison`, prints a side-by-side table. When multiple files are passed, output paths are auto-suffixed per eval name. +**`src/eval/index.ts`** — CLI entry point. Parses `--refine`, `--max-iter`, `--models`, `--cases`, `--output-json`, `--output-csv`, and `--history` flags. Accepts one or more eval file paths as positional arguments. `--cases` accepts comma-separated case IDs or 1-based index ranges (e.g. `simple,1-3`) to run a subset without editing YAML. Single-model mode: loads existing CSV results for resume (skips already-scored cases), runs remaining cases, optional refine loop. Multi-model mode (2+ models via `--models`): runs each model independently, builds an `EvalComparison`, prints a side-by-side table. When multiple files are passed, output paths are auto-suffixed per eval name. Every `EvalComparison` carries a `provenance` record (see `src/eval/provenance.ts`); `--history ` appends it (plus score/cost/duration) to a JSONL log for `npm run eval:trend`. **`src/eval/load.ts`** — Parses `evals/*.eval.yaml` via Zod. Resolves fixture paths (values in `vars` that end in `.md` / `.txt` are read and substituted with file contents). -**`src/eval/runner.ts`** — `runPrompt(templatePath, vars, model?)`: substitutes `{{PLACEHOLDER}}` vars, runs the prompt through the specified model via `runAgent`, and returns the raw text output. Claude receives `METHODOLOGY` as `appendSystemPrompt`; OpenCode does not (flag not supported). +**`src/eval/runner.ts`** — `runPrompt(templatePath, vars, model?)`: substitutes `{{PLACEHOLDER}}` vars, runs the prompt through the specified model via `runAgent`, and returns `{ output, costUsd? }` — the raw text output plus API cost when the provider reports one (Claude only). Claude receives `METHODOLOGY` as `appendSystemPrompt`; OpenCode does not (flag not supported). **`src/eval/judge.ts`** — `judgeOutput()`: takes a single output string and a criterion string, always uses Claude for judgment (the authoritative judge), and returns `{ pass: boolean, reason: string }`. **`src/eval/refine.ts`** — `refinePrompt()`: given the current template and a list of failures, calls Claude with the prompt-refiner prompt and returns a rewritten template. -**`src/eval/report.ts`** — Terminal output: `printRun()` for single-model pass/fail table; `printComparison()` for multi-model side-by-side comparison table. +**`src/eval/provenance.ts`** — `buildProvenance(evalFile)`: captures `RunProvenance` for a comparison run — `git rev-parse HEAD` / origin remote for repo+SHA, `claude --version` for judge version, and sha256 hashes (truncated to 12 hex chars) of the judge prompt template and the resolved eval spec. `comparisonFingerprint` combines judge provider+model+prompt hash+eval hash into the key that decides whether two runs are strictly comparable. -**`src/eval/export.ts`** — `toJson(comparison)` and `toCsv(comparison)`: serialize `EvalComparison` for benchmark analysis. CSV is denormalized (one row per criterion judgment per model) with columns `eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms`. +**`src/eval/history.ts`** — `appendHistory`/`loadHistory`: persist one JSONL record per model/eval per run (score, cost, duration, provenance) to a history log. `buildTrends(entries, mode)`: groups records by eval+model into time-ordered series; `"strict"` keeps only runs matching the group's latest `comparisonFingerprint`, `"all"` keeps every run and flags the points where the fingerprint changed (`regimeChange`). + +**`src/eval/trend-index.ts`** — `npm run eval:trend` CLI. Reads a history JSONL file, filters by `--eval`, builds trends in `--mode strict|all`, and renders them via `printTrends`. + +**`src/eval/report.ts`** — Terminal output: `printRun()` for single-model pass/fail table; `printComparison()` for multi-model side-by-side comparison table; `printTrends()` for the `eval:trend` time-series view, with a marker line at each regime-change point. + +**`src/eval/export.ts`** — `toJson(comparison)` and `toCsv(comparison)`: serialize `EvalComparison` for benchmark analysis. CSV is denormalized (one row per criterion judgment per model) with columns `eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms, cost_usd, run_at, repo, git_sha, judge_provider, judge_model, judge_version, judge_prompt_hash, eval_hash, comparison_fingerprint`. **`src/eval/prompts/`** — Eval-specific prompts (`criterion-judge.txt`, `prompt-refiner.txt`). Same `{{PLACEHOLDER}}` convention as `src/prompts/`. diff --git a/BACKLOG.md b/BACKLOG.md index 32faa13..bf8e101 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -28,6 +28,18 @@ Known improvements deferred from code reviews and audits. - **Vars-aware caching in `resolveWorkflow`** — two `workflow:` steps referencing the same file are currently fetched and parsed independently. A cache would need to key on the resolved path/URL *and* the step's `vars:` override (serialized), since two steps referencing the same file with different overrides must not share a resolved `Workflow` — added complexity that isn't justified without evidence repeated-reference workflows are common. +## Implemented (status bar, 2026-08) + +- ✅ **Status bar wraps to a second line under a long repo name or branch** — `StatusBar` (`src/ui/StatusBar.tsx`) rendered the repo name and branch at their full length with no regard for terminal width; a sufficiently long value (a long feature-branch name, or a checkout directory not named after the repo) wrapped the row onto a second line, breaking the fixed single-line layout the rest of the footer assumes. `fitRepoLabel` (`src/lib/statusline.ts`) now shrinks name and/or branch with an ellipsis to fit the terminal width minus the gauge segment's reserved width, handing a short name's unused budget to the branch. `App.tsx` passes `stdout?.columns ?? 80` down as the new required `columns` prop. Caught via `src/tests/statusline-ui.test.ts`, whose "names the repo and branch" test also hardcoded the literal string "executant" as the expected repo name (true only when the checkout directory happens to be named after the repo) — updated to assert the shape of the segment rather than its exact text, so the suite doesn't assume a specific checkout location. + +## Implemented (subprocess lifecycle, 2026-08) + +- ✅ **`timeout_seconds` on a script step actually kills the command on dash-family shells** — `runCommand` (`src/tasks/command.ts`) spawns `sh -c ""` and, on timeout, called `proc.kill()` against that `sh` process alone. On shells that fork a real child for the command rather than exec-replacing themselves (verified with `dash`, Debian/Ubuntu's default `/bin/sh`, for anything beyond a single tail-callable command — e.g. `sleep 60`), killing only the `sh` PID left the actual command running as an orphan, still holding stdout/stderr open. The step's reader loop was waiting on EOF from that pipe, so it never saw one and hung indefinitely instead of throwing `TimeoutError` — silently defeating the feature on any Linux box where `/bin/sh` is dash (i.e. most). Fixed by spawning `detached: true` (making `proc.pid` the leader of its own process group) and killing the whole group (`process.kill(-proc.pid!, "SIGTERM")`) on timeout — `startTimeout` (`src/tasks/stream.ts`) now takes an optional `onTimeout` override for this, defaulting to the original `proc.kill()` for `claude.ts`/`opencode.ts`, which spawn the CLI binary directly and don't have this problem. Detaching moves the child out of executant's own process group, so `command.ts` also gained explicit `SIGINT`/`SIGTERM`/`SIGHUP` cleanup handlers (mirroring the pattern already used in `claude.ts`/`opencode.ts` for `SIGTERM`/`SIGHUP`) so a terminal Ctrl+C during a running script step still reaches the whole process tree instead of just executant itself. Caught via `src/tests/command.test.ts`'s existing timeout test, which used to hang for the full 60s test-runner timeout on this sandbox rather than failing fast — a new regression test spawns a real detached grandchild and asserts it's gone (not orphaned) after the timeout fires. + +## Implemented (eval comparison observability, 2026-08) + +- ✅ **Run provenance + judge drift markers for eval comparisons** — historical eval-comparison trends were previously opaque: a score change on the same eval/model could come from the model under test, or silently from a judge/prompt/eval regime change, with no way to tell which. Every `EvalComparison` (`src/eval/index.ts`) now carries a `provenance` record (`src/eval/provenance.ts`): `runAt`, `repo`/`gitSha` (from `git`), `judgeProvider`/`judgeModel`/`judgeVersion` (`claude --version`, when readable), and sha256 hashes of the judge prompt template and the resolved eval spec, combined into a `comparisonFingerprint` — the strict-comparability key. Per-case API cost (`TestResult.costUsd`, Claude only — OpenCode/local models don't report one) is captured alongside score and duration and rolled up into `EvalRun.totalCostUsd`. JSON/CSV output (`src/eval/export.ts`) include all of it; CSV repeats the values per row the same way `duration_ms` already did, keeping the pivot-table shape. `--history ` (new flag, wired into `eval:compare`) appends one JSONL record per model/eval per run (`src/eval/history.ts`); `npm run eval:trend` (`src/eval/trend-index.ts`) reads that log and renders per eval+model time series in two modes — `all` (every run, with a marker line wherever the judge/prompt/eval fingerprint changed since the previous run) and `strict` (only runs matching the latest fingerprint). The separate git-worktree-based workflow-eval harness (`src/eval/workflow.ts`) was left untouched — this only covers the criterion-judge comparison system `docs/eval-comparison.md` documents. + ## Implemented (TUI, 2026-08) - ✅ **Context gauge reads per-call usage, not the step's cumulative total** — the gauge shipped reading `output:usage`, which is parsed from the Claude CLI's final `result` message. Those counts are summed across every API call the step made: each turn re-reads the whole cached prefix, so `cache_read_input_tokens` grows by roughly the context size *per turn*. Dividing that by a 200k window is a category error — it compares throughput to capacity. On a long agentic step it read **3781.1k/200k** (~25 turns' worth), with the bar pinned at 100% because `buildGauge` caps `pct`. Verified against a real three-turn `claude -p` transcript: the calls occupied 37,670 → 37,844 → 38,166 tokens while the result message reported 113,680 — exactly the sum of the three. The fix adds a separate `output:context` event emitted from the stream's `assistant` messages (deduplicated by message id, since the CLI emits one per content block sharing that turn's usage) carrying the session's `input + cacheCreation + cacheRead` as of its latest turn; the gauge replaces on each one, and `output:usage` goes back to being purely the run report's throughput total. The gauge is also cleared on a prompt step's `step:start`, so each step's session starts from an empty window rather than inheriting the previous step's fill — one session per step, one gauge per session, never carried across or summed. A side benefit is that the gauge now moves *during* a step rather than only at its end. diff --git a/README.md b/README.md index 979cee4..a167982 100644 --- a/README.md +++ b/README.md @@ -580,7 +580,7 @@ npm run eval -- \ npm run eval -- evals/plan-decompose.eval.yaml evals/judge-evaluation.eval.yaml ``` -The `--output-csv` file is denormalized (one row per criterion judgment per model) — ready for pivot tables and charts. See [docs/eval-comparison.md](docs/eval-comparison.md) for column definitions and interpretation guidance. +The `--output-csv` file is denormalized (one row per criterion judgment per model) — ready for pivot tables and charts. Every run also carries provenance (repo, git SHA, judge model/prompt, eval spec hash) and per-case cost. Pass `--history results/eval-history.jsonl` to accumulate a time series, then `npm run eval:trend` to view it, with markers wherever the judge/prompt/eval config changed. See [docs/eval-comparison.md](docs/eval-comparison.md) for column definitions and interpretation guidance. ### Workflow evals (end-to-end agentic testing) diff --git a/docs/eval-comparison.md b/docs/eval-comparison.md index 754edc9..3aa5ed5 100644 --- a/docs/eval-comparison.md +++ b/docs/eval-comparison.md @@ -63,14 +63,47 @@ judge-evaluation — 2 models compared TOTAL 7/9 78% 8/9 89% ``` +## Run provenance and cost + +Every comparison run captures a `provenance` record so historical trends stay +interpretable — a score change can come from the model under test, or from a +change in the judge/eval regime itself, and the two should never be confused: + +| Field | Description | +|---|---| +| `runAt` | ISO timestamp of the run | +| `repo` | `owner/repo`, parsed from the `origin` git remote (GitHub only) | +| `gitSha` | Commit evaluated (`git rev-parse HEAD`) | +| `judgeProvider` / `judgeModel` | The judge is always Claude — this records which model | +| `judgeVersion` | `claude --version`, when it can be read | +| `judgePromptHash` | Hash of `src/eval/prompts/criterion-judge.txt` | +| `evalHash` | Hash of the resolved eval spec (test cases, vars, criteria) | +| `comparisonFingerprint` | Hash of judge provider+model+prompt hash+eval hash — the strict-comparability key | + +Each case's API cost (USD, Claude only — OpenCode/local models don't report +cost) is captured alongside its score and duration on every `TestResult`, and +rolled up into `EvalRun.totalCostUsd`. + ## JSON output format -The `--output-json` file contains the full `EvalComparison` object: +The `--output-json` file contains the full `EvalComparison` object, including +`provenance`: ```json { "evalName": "judge-evaluation", "templatePath": "evals/judge-evaluation.eval.yaml", + "provenance": { + "runAt": "2026-08-31T12:00:00.000Z", + "repo": "coston/executant", + "gitSha": "4c7875ebb732337aba1737e357f2b7ba51f064e2", + "judgeProvider": "claude", + "judgeModel": "sonnet", + "judgeVersion": "2.1.251", + "judgePromptHash": "a1b2c3d4e5f6", + "evalHash": "f6e5d4c3b2a1", + "comparisonFingerprint": "0011223344aa" + }, "models": [ { "provider": "claude", "model": "sonnet" }, { "provider": "opencode", "model": "llama-qwen7b/qwen2.5-coder-7b" } @@ -123,13 +156,20 @@ The `--output-csv` file is **denormalized** — one row per criterion judgment p | `model` | Model name as passed to the CLI | | `pass` | `true` or `false` | | `reason` | Judge's reasoning for the pass/fail verdict | +| `duration_ms` | Wall-clock time for the case | +| `cost_usd` | API cost for the case (Claude only; empty for OpenCode/local models) | +| `run_at`, `repo`, `git_sha`, `judge_provider`, `judge_model`, `judge_version`, `judge_prompt_hash`, `eval_hash`, `comparison_fingerprint` | Run provenance — see [Run provenance and cost](#run-provenance-and-cost) | + +Provenance and cost values repeat across every row of a run, the same way +`duration_ms` already does — the CSV is denormalized for pivot tables, not +optimized for storage. ### Example rows ```csv -eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason -"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","claude/sonnet","claude","sonnet","true","Response is well-formed JSON" -"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","opencode/llama-qwen7b/qwen2.5-coder-7b","opencode","llama-qwen7b/qwen2.5-coder-7b","true","JSON parses without error" +eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms,cost_usd,run_at,repo,git_sha,judge_provider,judge_model,judge_version,judge_prompt_hash,eval_hash,comparison_fingerprint +"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","claude/sonnet","claude","sonnet","true","Response is well-formed JSON",1820,0.0042,"2026-08-31T12:00:00.000Z","coston/executant","4c7875ebb732337aba1737e357f2b7ba51f064e2","claude","sonnet","2.1.251","a1b2c3d4e5f6","f6e5d4c3b2a1","0011223344aa" +"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","opencode/llama-qwen7b/qwen2.5-coder-7b","opencode","llama-qwen7b/qwen2.5-coder-7b","true","JSON parses without error",4310,"","2026-08-31T12:00:00.000Z","coston/executant","4c7875ebb732337aba1737e357f2b7ba51f064e2","claude","sonnet","2.1.251","a1b2c3d4e5f6","f6e5d4c3b2a1","0011223344aa" ``` ### Pivot table recipe (Excel / Google Sheets) @@ -142,6 +182,43 @@ eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason Plot `model_label` on X axis, `pct = pass / total_per_model` on Y axis, grouped by `eval_name`. This gives a quick overview of relative model performance across prompt templates. +## Historical trend reporting + +Pass `--history ` to append one JSONL record per model/eval to a history +log every time you run an eval — score, cost, duration, and the provenance +above (this is what `eval:compare` does automatically, into +`results/eval-history.jsonl`): + +```bash +npm run eval -- --models claude/sonnet,claude/haiku \ + --history results/eval-history.jsonl \ + evals/judge-evaluation.eval.yaml +``` + +Then view the trend for each eval+model series: + +```bash +npm run eval:trend # all history, all runs +npm run eval:trend -- --eval judge-evaluation # filter to one eval +npm run eval:trend -- --mode strict # only strictly-comparable runs +npm run eval:trend -- --history results/other.jsonl # a different history file +``` + +Two modes: + +- **`all`** (default) — every historical run is shown, so no data is ever lost. A + `─── regime change ───` marker line is printed wherever the judge model/version, + judge prompt, or eval spec (`comparison_fingerprint`) changed since the + previous run in that series, so a score jump reads as a regime change rather + than a mysterious improvement or regression. +- **`strict`** — only runs whose `comparison_fingerprint` matches the most + recent run are shown. This is the safe default for "is the model actually + getting better/worse" questions, at the cost of dropping older runs made + under a different judge/prompt/eval config. + +Each trend point can always be traced back to its `run_at`, `git_sha`, and +full judge config via the underlying history record. + ## Adding a new model Any provider supported by Executant can be added to a comparison run: diff --git a/package.json b/package.json index 98ffc24..7c51916 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,13 @@ "test": "env -u NODE_TEST_CONTEXT -u EXECUTANT_PROVIDER -u EXECUTANT_MODEL -u EXECUTANT_AGENT -u EXECUTANT_REPORT_SUGGESTION -u OTEL_EXPORTER_OTLP_ENDPOINT -u OTEL_EXPORTER_OTLP_TRACES_ENDPOINT -u OTEL_EXPORTER_OTLP_METRICS_ENDPOINT -u TRACEPARENT EXECUTANT_RETROSPECTIVE=0 EXECUTANT_STATUSLINE=0 node --import tsx/esm --test --test-timeout=60000 src/tests/*.test.ts", "eval": "tsx src/eval/index.ts", "eval:workflow": "tsx src/eval/workflow-index.ts", + "eval:trend": "tsx src/eval/trend-index.ts", "setup": "tsx src/setup.ts", "models:download": "tsx src/native-models.ts", "models:start": "tsx src/model-server.ts start", "models:stop": "tsx src/model-server.ts stop", "models:status": "tsx src/model-server.ts status", - "eval:compare": "for f in evals/*.eval.yaml; do npm run eval -- --models claude/opus,claude/sonnet,claude/haiku,opencode/llama-qwen7b/qwen2.5-coder-7b,opencode/llama-qwen14b/qwen2.5-coder-14b,opencode/llama-llama8b/llama-3.1-8b --output-csv \"results/$(basename $f .eval.yaml).csv\" \"$f\"; done && npm run eval:compare:report", + "eval:compare": "for f in evals/*.eval.yaml; do npm run eval -- --models claude/opus,claude/sonnet,claude/haiku,opencode/llama-qwen7b/qwen2.5-coder-7b,opencode/llama-qwen14b/qwen2.5-coder-14b,opencode/llama-llama8b/llama-3.1-8b --output-csv \"results/$(basename $f .eval.yaml).csv\" --history results/eval-history.jsonl \"$f\"; done && npm run eval:compare:report", "eval:compare:report": "tsx src/eval/report-gen.ts", "lint": "eslint src", "knip": "knip" @@ -116,7 +117,8 @@ "src/model-server.ts", "src/eval/index.ts", "src/eval/workflow-index.ts", - "src/eval/report-gen.ts" + "src/eval/report-gen.ts", + "src/eval/trend-index.ts" ], "project": [ "src/**/*.ts", diff --git a/src/eval/export.ts b/src/eval/export.ts index e59dcb6..a82120d 100644 --- a/src/eval/export.ts +++ b/src/eval/export.ts @@ -4,9 +4,15 @@ // Serializes EvalComparison results to JSON and CSV for benchmark analysis. // // CSV columns (one row per criterion judgment): -// eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms +// eval_name, template_path, case_id, criterion, model_label, provider, model, +// pass, reason, duration_ms, cost_usd, run_at, repo, git_sha, judge_provider, +// judge_model, judge_version, judge_prompt_hash, eval_hash, comparison_fingerprint +// +// Provenance and cost columns repeat the same value across every row of a run +// (like duration_ms already does) — the denormalized shape is optimized for +// pivot tables, not storage efficiency. -import type { EvalComparison, ModelTarget } from "./types.js"; +import type { EvalComparison, ModelTarget, RunProvenance } from "./types.js"; export function modelLabel(m: ModelTarget): string { return m.label ?? `${m.provider}/${m.model}`; @@ -17,6 +23,32 @@ export function toJson(comparison: EvalComparison): string { return JSON.stringify(comparison, null, 2); } +const PROVENANCE_COLUMNS = [ + "run_at", + "repo", + "git_sha", + "judge_provider", + "judge_model", + "judge_version", + "judge_prompt_hash", + "eval_hash", + "comparison_fingerprint", +] as const; + +function provenanceCells(p: RunProvenance): string[] { + return [ + csvCell(p.runAt), + csvCell(p.repo ?? ""), + csvCell(p.gitSha ?? ""), + csvCell(p.judgeProvider), + csvCell(p.judgeModel), + csvCell(p.judgeVersion ?? ""), + csvCell(p.judgePromptHash), + csvCell(p.evalHash), + csvCell(p.comparisonFingerprint), + ]; +} + /** Serializes a comparison to CSV — one row per criterion judgment per model. */ export function toCsv(comparison: EvalComparison): string { const header = [ @@ -30,6 +62,8 @@ export function toCsv(comparison: EvalComparison): string { "pass", "reason", "duration_ms", + "cost_usd", + ...PROVENANCE_COLUMNS, ].join(","); const rows: string[] = [header]; @@ -50,6 +84,8 @@ export function toCsv(comparison: EvalComparison): string { c.pass ? "true" : "false", csvCell(c.reason), String(result.durationMs), + result.costUsd !== undefined ? String(result.costUsd) : "", + ...provenanceCells(comparison.provenance), ].join(","), ); } diff --git a/src/eval/history.ts b/src/eval/history.ts new file mode 100644 index 0000000..b27bda2 --- /dev/null +++ b/src/eval/history.ts @@ -0,0 +1,143 @@ +// ============================================================================ +// EVAL HISTORY +// ============================================================================ +// Persists one JSONL record per model/eval per comparison run (score, cost, +// duration, provenance) so trends can be tracked over time, and groups those +// records into per eval+model trend lines for `npm run eval:trend`. +// +// Two trend modes: +// "strict" — only runs whose comparisonFingerprint matches the most recent +// run in the group (judge/prompt/eval unchanged) +// "all" — every historical run, with regime-change points flagged so +// judge/prompt/eval drift is explicit rather than hidden + +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { modelLabel } from "./export.js"; +import type { EvalComparison } from "./types.js"; + +interface HistoryEntry { + runAt: string; + repo?: string; + gitSha?: string; + evalName: string; + modelLabel: string; + provider: string; + model: string; + passCount: number; + totalCriteria: number; + pct: number; + costUsd?: number; + durationMs: number; + judgeProvider: string; + judgeModel: string; + judgeVersion?: string; + judgePromptHash: string; + evalHash: string; + comparisonFingerprint: string; +} + +/** One row per model run in the comparison, carrying the run's shared provenance. */ +export function toHistoryEntries(comparison: EvalComparison): HistoryEntry[] { + const { provenance } = comparison; + return comparison.runs.map((run) => ({ + runAt: provenance.runAt, + repo: provenance.repo, + gitSha: provenance.gitSha, + evalName: comparison.evalName, + modelLabel: modelLabel(run.model), + provider: run.model.provider, + model: run.model.model, + passCount: run.totalPass, + totalCriteria: run.totalCriteria, + pct: run.totalCriteria === 0 ? 0 : run.totalPass / run.totalCriteria, + costUsd: run.totalCostUsd, + durationMs: run.results.reduce((s, r) => s + r.durationMs, 0), + judgeProvider: provenance.judgeProvider, + judgeModel: provenance.judgeModel, + judgeVersion: provenance.judgeVersion, + judgePromptHash: provenance.judgePromptHash, + evalHash: provenance.evalHash, + comparisonFingerprint: provenance.comparisonFingerprint, + })); +} + +/** Appends one JSONL line per model run in the comparison to `historyPath`. */ +export function appendHistory( + comparison: EvalComparison, + historyPath: string, +): void { + const entries = toHistoryEntries(comparison); + if (entries.length === 0) return; + mkdirSync(dirname(historyPath), { recursive: true }); + const lines = entries.map((e) => JSON.stringify(e)).join("\n") + "\n"; + appendFileSync(historyPath, lines, "utf8"); +} + +/** Reads all history records from a JSONL file. Returns [] if the file doesn't exist. */ +export function loadHistory(historyPath: string): HistoryEntry[] { + if (!existsSync(historyPath)) return []; + return readFileSync(historyPath, "utf8") + .split("\n") + .filter((line) => line.trim()) + .map((line) => JSON.parse(line) as HistoryEntry); +} + +export type TrendMode = "strict" | "all"; + +export interface TrendPoint extends HistoryEntry { + /** True when this point's comparisonFingerprint differs from the previous point in its group. */ + regimeChange: boolean; +} + +export interface TrendGroup { + evalName: string; + modelLabel: string; + points: TrendPoint[]; +} + +/** + * Groups history entries by eval+model into time-ordered trend lines. + * "strict" keeps only runs matching the group's most recent comparisonFingerprint + * (guaranteed judge/prompt/eval comparability); "all" keeps every run and marks + * the points where the fingerprint changed. + */ +export function buildTrends( + entries: HistoryEntry[], + mode: TrendMode, +): TrendGroup[] { + const byGroup = new Map(); + for (const entry of entries) { + const key = `${entry.evalName}::${entry.modelLabel}`; + const group = byGroup.get(key); + if (group) group.push(entry); + else byGroup.set(key, [entry]); + } + + const groups: TrendGroup[] = []; + for (const [key, groupEntries] of byGroup) { + const [evalName, label] = key.split("::") as [string, string]; + const sorted = [...groupEntries].sort((a, b) => + a.runAt.localeCompare(b.runAt), + ); + const latestFingerprint = sorted.at(-1)?.comparisonFingerprint; + const selected = + mode === "strict" + ? sorted.filter((e) => e.comparisonFingerprint === latestFingerprint) + : sorted; + + const points: TrendPoint[] = selected.map((entry, i) => ({ + ...entry, + regimeChange: + i > 0 && + selected[i - 1]!.comparisonFingerprint !== entry.comparisonFingerprint, + })); + groups.push({ evalName, modelLabel: label, points }); + } + + return groups.sort( + (a, b) => + a.evalName.localeCompare(b.evalName) || + a.modelLabel.localeCompare(b.modelLabel), + ); +} diff --git a/src/eval/index.ts b/src/eval/index.ts index eeb3332..411faaa 100644 --- a/src/eval/index.ts +++ b/src/eval/index.ts @@ -29,6 +29,8 @@ import { printDiff, } from "./report.js"; import { toJson, toCsv, modelLabel } from "./export.js"; +import { buildProvenance } from "./provenance.js"; +import { appendHistory } from "./history.js"; import type { EvalArgs, EvalRun, @@ -37,6 +39,7 @@ import type { FailureContext, ModelTarget, ModelEvalRun, + RunProvenance, TestResult, } from "./types.js"; @@ -101,6 +104,9 @@ export function loadExistingResults( const pass = cells[col["pass"]] === "true"; const reason = cells[col["reason"]] ?? ""; const durationMs = parseInt(cells[col["duration_ms"]] ?? "0", 10); + const costCell = cells[col["cost_usd"]]; + const costUsd = + costCell !== undefined && costCell !== "" ? Number(costCell) : undefined; if (!byModel.has(label)) byModel.set(label, new Map()); const byCase = byModel.get(label)!; @@ -113,6 +119,7 @@ export function loadExistingResults( passCount: 0, failCount: 0, durationMs, + costUsd, }); } const result = byCase.get(caseId)!; @@ -198,6 +205,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { let outputJson: string | undefined; let outputCsv: string | undefined; let caseFilter: string | undefined; + let historyPath: string | undefined; for (let i = 0; i < rawArgs.length; i++) { const arg = rawArgs[i]!; @@ -215,6 +223,8 @@ export function parseArgs(rawArgs: string[]): EvalArgs { outputCsv = rawArgs[++i]; } else if (arg === "--cases" && rawArgs[i + 1]) { caseFilter = rawArgs[++i]; + } else if (arg === "--history" && rawArgs[i + 1]) { + historyPath = rawArgs[++i]; } else if (!arg.startsWith("-")) { evalFiles.push(arg); } @@ -232,6 +242,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { " --cases Run a subset of cases: IDs or index ranges, e.g. simple,1-3", " --output-json Write comparison JSON to file", " --output-csv Write comparison CSV to file (supports resume)", + " --history Append a JSONL history record for trend tracking (see `npm run eval:trend`)", ].join("\n"), ); process.exit(0); @@ -251,6 +262,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { models, outputJson, outputCsv, + historyPath, }; } @@ -281,8 +293,9 @@ async function runEval( process.stdout.write(` running ${tc.id}…`); const start = performance.now(); let output: string; + let costUsd: number | undefined; try { - output = await runPrompt(path, tc.vars, model); + ({ output, costUsd } = await runPrompt(path, tc.vars, model)); } catch (err) { const durationMs = Math.round(performance.now() - start); const msg = `run error: ${err instanceof Error ? err.message : String(err)}`; @@ -313,12 +326,18 @@ async function runEval( passCount, failCount, durationMs, + costUsd, }); process.stdout.write(` ${passCount}/${criteria.length}\n`); } const totalPass = results.reduce((s, r) => s + r.passCount, 0); const totalCriteria = results.reduce((s, r) => s + r.criteria.length, 0); + const costs = results + .map((r) => r.costUsd) + .filter((c): c is number => typeof c === "number"); + const totalCostUsd = + costs.length > 0 ? costs.reduce((a, b) => a + b, 0) : undefined; return { evalName: evalFile.name, @@ -326,6 +345,7 @@ async function runEval( results, totalPass, totalCriteria, + totalCostUsd, }; } @@ -380,6 +400,7 @@ function buildComparisonTable( async function runMultiModelEval( evalFile: ReturnType, models: ModelTarget[], + provenance: RunProvenance, existingCsv?: string, caseFilter?: string, ): Promise { @@ -405,6 +426,7 @@ async function runMultiModelEval( models, runs, comparisonTable: buildComparisonTable(runs), + provenance, }; } @@ -463,6 +485,8 @@ async function runEvalFile( ? deriveOutputPath(args.outputJson, evalFile.name) : args.outputJson; + const provenance = buildProvenance(evalFile); + // Multi-model comparison mode if (args.models.length > 1) { if (args.refine) { @@ -473,6 +497,7 @@ async function runEvalFile( const comparison = await runMultiModelEval( evalFile, args.models, + provenance, outputCsv, args.caseFilter, ); @@ -480,6 +505,7 @@ async function runEvalFile( if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); + if (args.historyPath) appendHistory(comparison, args.historyPath); return; } @@ -497,7 +523,7 @@ async function runEvalFile( printRun(run); // Write output files (wraps single-model run in a minimal comparison) - if (outputJson || outputCsv) { + if (outputJson || outputCsv || args.historyPath) { const model = singleModel ?? { provider: "claude" as const, model: "sonnet", @@ -508,9 +534,11 @@ async function runEvalFile( models: [model], runs: [{ ...run, model }], comparisonTable: buildComparisonTable([{ ...run, model }]), + provenance, }; if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); + if (args.historyPath) appendHistory(comparison, args.historyPath); } if (!args.refine || run.totalPass === run.totalCriteria) return; diff --git a/src/eval/provenance.ts b/src/eval/provenance.ts new file mode 100644 index 0000000..09e6c52 --- /dev/null +++ b/src/eval/provenance.ts @@ -0,0 +1,115 @@ +// ============================================================================ +// EVAL PROVENANCE +// ============================================================================ +// Captures the "what produced this score" context for a comparison run: +// repo + commit evaluated, judge identity, and hashes of the judge prompt and +// eval spec. `comparisonFingerprint` is the stable key that tells historical +// trend reporting whether two runs are strictly comparable. + +import { execSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { DEFAULT_MODEL } from "../lib/utils.js"; +import type { EvalFile, RunProvenance } from "./types.js"; + +const __dir = dirname(fileURLToPath(import.meta.url)); + +function tryExec(cmd: string): string | undefined { + try { + const out = execSync(cmd, { stdio: ["ignore", "pipe", "ignore"] }) + .toString() + .trim(); + return out || undefined; + } catch { + return undefined; + } +} + +/** Commit SHA currently checked out, or undefined outside a git repo. */ +export function getGitSha(): string | undefined { + return tryExec("git rev-parse HEAD"); +} + +/** "owner/repo" parsed from the origin remote URL (GitHub only), or undefined. */ +export function getRepoSlug(): string | undefined { + const url = tryExec("git config --get remote.origin.url"); + if (!url) return undefined; + const match = /github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?$/.exec(url); + return match?.[1]; +} + +const UNSET = Symbol("unset"); +let cachedJudgeVersion: string | undefined | typeof UNSET = UNSET; + +/** + * The judge CLI's version string, when it can be read (e.g. "2.1.251"). + * Memoized per process — every eval-comparison run only pays for one + * `claude --version` spawn no matter how many eval files it covers. + */ +function getJudgeVersion(): string | undefined { + if (cachedJudgeVersion !== UNSET) return cachedJudgeVersion; + const out = tryExec("claude --version"); + cachedJudgeVersion = out?.split(/\s+/)[0]; + return cachedJudgeVersion; +} + +/** Stable short hash (sha256, truncated) of arbitrary content. */ +export function hashContent(content: string): string { + return createHash("sha256").update(content).digest("hex").slice(0, 12); +} + +/** Hash of the judge prompt template — changes here signal a judging-regime change. */ +function getJudgePromptHash(): string { + const promptPath = join(__dir, "prompts", "criterion-judge.txt"); + return hashContent(readFileSync(promptPath, "utf8")); +} + +/** Hash of the resolved eval spec (test cases + criteria) — changes here signal an eval-regime change. */ +export function getEvalHash(evalFile: EvalFile): string { + const canonical = JSON.stringify({ + name: evalFile.name, + placeholders: [...evalFile.placeholders].sort(), + testCases: [...evalFile.testCases] + .sort((a, b) => a.id.localeCompare(b.id)) + .map((tc) => ({ id: tc.id, vars: tc.vars, criteria: tc.criteria })), + }); + return hashContent(canonical); +} + +/** Stable fingerprint of judge+prompt+eval config — the strict-comparability key. */ +export function computeComparisonFingerprint( + judgeProvider: string, + judgeModel: string, + judgePromptHash: string, + evalHash: string, +): string { + return hashContent( + `${judgeProvider}:${judgeModel}:${judgePromptHash}:${evalHash}`, + ); +} + +/** Builds the full provenance record for a comparison run over the given eval file. */ +export function buildProvenance(evalFile: EvalFile): RunProvenance { + const judgeProvider = "claude"; + const judgeModel = process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL; + const judgePromptHash = getJudgePromptHash(); + const evalHash = getEvalHash(evalFile); + return { + runAt: new Date().toISOString(), + repo: getRepoSlug(), + gitSha: getGitSha(), + judgeProvider, + judgeModel, + judgeVersion: getJudgeVersion(), + judgePromptHash, + evalHash, + comparisonFingerprint: computeComparisonFingerprint( + judgeProvider, + judgeModel, + judgePromptHash, + evalHash, + ), + }; +} diff --git a/src/eval/report.ts b/src/eval/report.ts index 5e900d4..35889bf 100644 --- a/src/eval/report.ts +++ b/src/eval/report.ts @@ -1,6 +1,8 @@ import type { EvalComparison, EvalRun, TestResult } from "./types.js"; +import type { TrendGroup } from "./history.js"; import { modelLabel } from "./export.js"; import { theme } from "../ui/theme.js"; +import { formatDuration } from "../lib/utils.js"; const USE_COLOR = Boolean(process.stdout.isTTY) && !process.env["NO_COLOR"]; @@ -159,3 +161,40 @@ export function printComparison(comparison: EvalComparison): void { }); console.log(` ${"TOTAL".padEnd(caseColWidth)} ${totalCells.join("")}\n`); } + +/** + * Prints time-series trend lines for `npm run eval:trend` — one section per + * eval+model group, oldest run first, with a marker row wherever the judge + * model/version, judge prompt, or eval spec changed since the previous run + * (so score jumps caused by a regime change read as a regime change, not a + * mysterious improvement or regression). + */ +export function printTrends(groups: TrendGroup[]): void { + for (const group of groups) { + console.log( + `\n${accent(group.evalName)} — ${dim(group.modelLabel)} (${group.points.length} run(s))\n`, + ); + for (const point of group.points) { + if (point.regimeChange) { + console.log( + warning( + " ─── regime change: judge/prompt/eval fingerprint differs from previous run ───", + ), + ); + } + const pctInt = Math.round(point.pct * 100); + const colorFn = + point.pct === 1 ? pass : point.pct >= 0.5 ? warning : fail; + const score = colorFn( + `${point.passCount}/${point.totalCriteria} ${pctInt}%`, + ); + const cost = + point.costUsd !== undefined ? `$${point.costUsd.toFixed(4)}` : "n/a"; + const sha = point.gitSha ? point.gitSha.slice(0, 7) : "unknown"; + console.log( + ` ${dim(point.runAt)} ${accent(sha)} ${score} ${dim(cost)} ${dim(formatDuration(point.durationMs))}`, + ); + } + } + console.log(); +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts index ce31249..5421d2e 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -18,16 +18,23 @@ export function substituteVars( ); } +/** Result of running a prompt template through a model. */ +interface PromptRunResult { + output: string; + /** API cost in USD, when the provider reports it (Claude only — OpenCode/local models don't). */ + costUsd?: number; +} + /** * Runs a prompt template with substituted vars through the specified model (no tools). * Defaults to Claude/sonnet when no model target is provided. - * Returns the full text output as a string. + * Returns the full text output plus cost, when the provider reports one. */ export async function runPrompt( templatePath: string, vars: Record, model?: ModelTarget, -): Promise { +): Promise { const template = stripPromptHeader(readFileSync(templatePath, "utf8")); const prompt = substituteVars(template, vars); @@ -35,6 +42,7 @@ export async function runPrompt( const isOpenCode = provider === "opencode"; const lines: string[] = []; + let costUsd: number | undefined; for await (const event of runAgent({ type: "claude", name: `eval:${basename(templatePath, ".txt")}`, @@ -52,7 +60,8 @@ export async function runPrompt( ...(!isOpenCode ? { appendSystemPrompt: METHODOLOGY } : {}), })) { if (event.type === "output:text") lines.push(event.text); + else if (event.type === "output:cost") costUsd = event.usd; } - return lines.join(""); + return { output: lines.join(""), costUsd }; } diff --git a/src/eval/trend-index.ts b/src/eval/trend-index.ts new file mode 100644 index 0000000..a652c58 --- /dev/null +++ b/src/eval/trend-index.ts @@ -0,0 +1,94 @@ +#!/usr/bin/env node +// ============================================================================ +// EVAL:TREND — Historical trend reporting over eval-history.jsonl +// ============================================================================ +// Usage: +// npm run eval:trend +// npm run eval:trend -- --history results/eval-history.jsonl +// npm run eval:trend -- --mode strict +// npm run eval:trend -- --eval judge-evaluation +// +// Reads the JSONL history log produced by `npm run eval -- --history ` +// and prints per eval+model time series with regime-change markers wherever +// the judge model/version, judge prompt, or eval spec changed between runs. + +import { fileURLToPath } from "node:url"; +import { loadHistory, buildTrends } from "./history.js"; +import { printTrends } from "./report.js"; +import type { TrendMode } from "./history.js"; + +const DEFAULT_HISTORY_PATH = "results/eval-history.jsonl"; + +interface TrendArgs { + historyPath: string; + mode: TrendMode; + evalFilter?: string; +} + +export function parseTrendArgs(rawArgs: string[]): TrendArgs { + let historyPath = DEFAULT_HISTORY_PATH; + let mode: TrendMode = "all"; + let evalFilter: string | undefined; + + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]!; + if (arg === "--help" || arg === "-h") { + console.log( + [ + "Usage: npm run eval:trend -- [OPTIONS]", + "", + "Options:", + ` --history JSONL history file to read (default: ${DEFAULT_HISTORY_PATH})`, + " --mode strict|all strict: only runs comparable to the latest judge/prompt/eval config", + " all (default): every run, with regime-change markers", + " --eval Filter to a single eval_name", + ].join("\n"), + ); + process.exit(0); + } else if (arg === "--history" && rawArgs[i + 1]) { + historyPath = rawArgs[++i]!; + } else if (arg === "--mode" && rawArgs[i + 1]) { + const value = rawArgs[++i]!; + if (value !== "strict" && value !== "all") { + throw new Error( + `Invalid --mode "${value}": expected "strict" or "all"`, + ); + } + mode = value; + } else if (arg === "--eval" && rawArgs[i + 1]) { + evalFilter = rawArgs[++i]; + } + } + + return { historyPath, mode, evalFilter }; +} + +export async function main(): Promise { + const args = parseTrendArgs(process.argv.slice(2)); + const entries = loadHistory(args.historyPath).filter( + (e) => !args.evalFilter || e.evalName === args.evalFilter, + ); + + if (entries.length === 0) { + console.log( + `No history found at ${args.historyPath}. Run evals with "--history ${args.historyPath}" to start tracking trends.`, + ); + return; + } + + const groups = buildTrends(entries, args.mode); + console.log( + `\nTrend report (${args.mode === "strict" ? "strict-comparable" : "all runs"} mode) — ${groups.length} eval+model series\n`, + ); + printTrends(groups); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error( + "eval:trend error:", + err instanceof Error ? err.message : String(err), + ); + process.exit(1); + }); +} diff --git a/src/eval/types.ts b/src/eval/types.ts index c288a3b..8fd7b02 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -24,6 +24,8 @@ export interface TestResult { passCount: number; failCount: number; durationMs: number; + /** API cost in USD for generating this case's output. Undefined when the provider doesn't report cost (e.g. OpenCode/local models). */ + costUsd?: number; } export interface EvalRun { @@ -32,6 +34,8 @@ export interface EvalRun { results: TestResult[]; totalPass: number; totalCriteria: number; + /** Sum of results[].costUsd. Undefined when no result reported a cost. */ + totalCostUsd?: number; } export interface FailureContext { @@ -60,6 +64,25 @@ export interface ComparisonRow { scores: Record; } +/** + * Provenance metadata for a single eval-comparison run, captured so historical + * trends stay interpretable: which repo/commit was evaluated, which judge + * (model + prompt) scored it, and which eval spec was run. `comparisonFingerprint` + * is a stable hash of judge+prompt+eval config — two runs only belong on the + * same "strict comparable" trend line when it matches. + */ +export interface RunProvenance { + runAt: string; // ISO timestamp + repo?: string; // "owner/repo", when a GitHub remote is configured + gitSha?: string; // commit evaluated + judgeProvider: string; + judgeModel: string; + judgeVersion?: string; // judge CLI version, when it can be determined + judgePromptHash: string; + evalHash: string; // hash of the resolved eval spec (test cases + criteria) + comparisonFingerprint: string; // hash of judgeProvider+judgeModel+judgePromptHash+evalHash +} + /** Full multi-model comparison result for a single eval file. */ export interface EvalComparison { evalName: string; @@ -67,6 +90,7 @@ export interface EvalComparison { models: ModelTarget[]; runs: ModelEvalRun[]; comparisonTable: ComparisonRow[]; + provenance: RunProvenance; } export interface EvalArgs { @@ -82,6 +106,8 @@ export interface EvalArgs { outputJson?: string; /** File path to write comparison CSV to (optional). */ outputCsv?: string; + /** File path to append a JSONL history record to, for trend tracking (optional). */ + historyPath?: string; } // --------------------------------------------------------------------------- diff --git a/src/lib/statusline.ts b/src/lib/statusline.ts index 2077f35..8f0357e 100644 --- a/src/lib/statusline.ts +++ b/src/lib/statusline.ts @@ -125,6 +125,39 @@ export interface RepoInfo { branch?: string; } +/** Truncates to fit, appending an ellipsis rather than silently cutting off. */ +function truncateText(s: string, maxWidth: number): string { + if (maxWidth <= 0) return ""; + if (s.length <= maxWidth) return s; + if (maxWidth === 1) return "…"; + return s.slice(0, maxWidth - 1) + "…"; +} + +/** Width of the 3-space gap StatusBar renders between the repo name and branch. */ +const REPO_BRANCH_GAP = 3; + +/** + * Shrinks repo name and branch to fit within `maxWidth` combined, so a long + * branch name (or, in an unusually-named checkout, a long directory name) + * never wraps the status bar onto a second line. A short name keeps its full + * length and any width it doesn't use is handed to the branch. + */ +export function fitRepoLabel(repo: RepoInfo, maxWidth: number): RepoInfo { + if (!repo.branch) return { name: truncateText(repo.name, maxWidth) }; + + const available = maxWidth - REPO_BRANCH_GAP; + if (repo.name.length + repo.branch.length <= available) return repo; + if (available <= 0) return { name: truncateText(repo.name, maxWidth) }; + + const half = Math.floor(available / 2); + const nameWidth = Math.min(repo.name.length, half); + const branchWidth = Math.max(1, available - nameWidth); + return { + name: truncateText(repo.name, nameWidth), + branch: truncateText(repo.branch, branchWidth), + }; +} + /** Runs a git subcommand in `cwd`, resolving undefined on any failure. */ function runGit( args: string[], diff --git a/src/tasks/command.ts b/src/tasks/command.ts index de60593..0902278 100644 --- a/src/tasks/command.ts +++ b/src/tasks/command.ts @@ -32,9 +32,32 @@ export async function* runCommand(task: CommandTask): AsyncGenerator { const proc = spawn("sh", ["-c", task.command], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, ...traceparentEnv() }, + // `sh -c ""` forks a real child for the command on shells like + // dash whenever it isn't the single tail-callable command in the script + // — killing just the sh PID then leaves that child running and holding + // stdout/stderr open, so a reader waiting on EOF never sees one (this is + // why a timed-out step used to hang instead of stopping). Detached makes + // proc.pid the leader of its own process group, so signalling -proc.pid + // reaches sh and every process it forked. + detached: true, }); - const timeout = startTimeout(proc, task.name, task.timeoutSeconds); + // Detaching moves the child out of executant's own process group, so it no + // longer receives a terminal's Ctrl+C (SIGINT) for free the way a + // non-detached child would — these mirror that by killing the group + // explicitly whenever executant itself is being torn down. + const killGroup = (): void => { + try { + process.kill(-proc.pid!, "SIGTERM"); + } catch { + /* already dead, or never got a pid */ + } + }; + process.once("SIGINT", killGroup); + process.once("SIGTERM", killGroup); + process.once("SIGHUP", killGroup); + + const timeout = startTimeout(proc, task.name, task.timeoutSeconds, killGroup); try { for await (const line of mergeStreamsToLines(proc.stdout!, proc.stderr!)) { @@ -53,5 +76,8 @@ export async function* runCommand(task: CommandTask): AsyncGenerator { } } finally { timeout.cancel(); + process.off("SIGINT", killGroup); + process.off("SIGTERM", killGroup); + process.off("SIGHUP", killGroup); } } diff --git a/src/tasks/stream.ts b/src/tasks/stream.ts index 0446312..4872d78 100644 --- a/src/tasks/stream.ts +++ b/src/tasks/stream.ts @@ -103,16 +103,23 @@ export function waitForExit(proc: ReturnType): Promise { * Arms a kill-on-timeout guard for a child process. * Call check() after waitForExit() to throw TimeoutError if the timer fired. * Call cancel() in a finally block to disarm the timer on normal completion. + * `onTimeout`, when given, replaces the default `proc.kill()` — e.g. to kill + * a whole process group instead of just the immediate child (see command.ts). */ export function startTimeout( proc: ReturnType, taskName: string, timeoutSeconds: number | undefined, + onTimeout?: () => void, ): { check: () => void; cancel: () => void } { if (timeoutSeconds == null) return { check: () => {}, cancel: () => {} }; let timedOut = false; const timer = setTimeout(() => { timedOut = true; + if (onTimeout) { + onTimeout(); + return; + } try { proc.kill(); } catch { diff --git a/src/tests/command.test.ts b/src/tests/command.test.ts index ef46eda..224c0ad 100644 --- a/src/tests/command.test.ts +++ b/src/tests/command.test.ts @@ -5,6 +5,9 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; +import { readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runCommand } from "../tasks/command.js"; import type { @@ -133,6 +136,38 @@ describe("runCommand — timeout_seconds", () => { assert.equal((error as TimeoutError).exitCode, 3); }); + test("kills the whole process group on timeout, not just the sh wrapper", async () => { + // Regression: `sh -c ""` forks a real child for the command + // itself on shells like dash whenever it isn't the single tail-callable + // command in the script. Killing only the `sh` PID used to leave that + // child (here, `sleep`) running and holding stdout open, so the reader + // never saw EOF and the step hung until the outer test-runner timeout + // rather than throwing TimeoutError. + const pidFile = join( + tmpdir(), + `executant-cmd-timeout-test-${process.pid}-${Date.now()}.pid`, + ); + const task: CommandTask = { + type: "command", + name: "nested-sleep", + command: `sleep 60 & echo $! > ${pidFile}; wait`, + timeoutSeconds: 0.1, + }; + try { + const { error } = await collectEventsExpectingError(task); + assert.ok(error instanceof TimeoutError); + + const grandchildPid = Number(readFileSync(pidFile, "utf8").trim()); + assert.throws( + () => process.kill(grandchildPid, 0), + /ESRCH/, + "the sleep grandchild should have been killed, not left orphaned", + ); + } finally { + rmSync(pidFile, { force: true }); + } + }); + test("does not throw TimeoutError when command completes before timeout", async () => { const task: CommandTask = { type: "command", diff --git a/src/tests/eval-comparison.test.ts b/src/tests/eval-comparison.test.ts index 15441b4..26c7025 100644 --- a/src/tests/eval-comparison.test.ts +++ b/src/tests/eval-comparison.test.ts @@ -20,6 +20,7 @@ import type { EvalComparison, ModelEvalRun, ModelTarget, + RunProvenance, } from "../eval/types.js"; // ---------------------------------------------------------------------------- @@ -121,6 +122,20 @@ describe("parseArgs — models / output flags", () => { assert.equal(args.outputCsv, undefined); }); + test("--history is parsed", () => { + const args = parseArgs([ + "--history", + "results/eval-history.jsonl", + "evals/test.yaml", + ]); + assert.equal(args.historyPath, "results/eval-history.jsonl"); + }); + + test("historyPath is undefined by default", () => { + const args = parseArgs(["evals/test.yaml"]); + assert.equal(args.historyPath, undefined); + }); + test("all new flags coexist with existing flags", () => { const args = parseArgs([ "--refine", @@ -175,6 +190,20 @@ describe("modelLabel", () => { // Fixture helpers // ---------------------------------------------------------------------------- +function makeProvenance(): RunProvenance { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "abc123def456abc123def456abc123def456abc", + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "hash-prompt-1", + evalHash: "hash-eval-1", + comparisonFingerprint: "hash-fingerprint-1", + }; +} + function makeComparison(): EvalComparison { const claudeModel: ModelTarget = { provider: "claude", model: "sonnet" }; const ocModel: ModelTarget = { @@ -277,6 +306,7 @@ function makeComparison(): EvalComparison { }, }, ], + provenance: makeProvenance(), }; } @@ -322,7 +352,7 @@ describe("toCsv", () => { const lines = csv.trim().split("\n"); assert.equal( lines[0], - "eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms", + "eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms,cost_usd,run_at,repo,git_sha,judge_provider,judge_model,judge_version,judge_prompt_hash,eval_hash,comparison_fingerprint", ); }); @@ -355,6 +385,31 @@ describe("toCsv", () => { const csv = toCsv(c); assert.ok(csv.includes('"failed, ""badly"""')); }); + + test("rows carry the comparison's provenance columns", () => { + const c = makeComparison(); + const csv = toCsv(c); + assert.ok(csv.includes('"coston/executant"')); + assert.ok(csv.includes('"abc123def456abc123def456abc123def456abc"')); + assert.ok(csv.includes('"hash-fingerprint-1"')); + }); + + test("cost_usd is empty when a result has no cost", () => { + const c = makeComparison(); + const csv = toCsv(c); + const lines = csv.trim().split("\n"); + // header: ...,duration_ms,cost_usd,run_at,... + const costIdx = lines[0]!.split(",").indexOf("cost_usd"); + const firstDataRow = lines[1]!.split(","); + assert.equal(firstDataRow[costIdx], ""); + }); + + test("cost_usd is serialized when a result reports a cost", () => { + const c = makeComparison(); + c.runs[0]!.results[0]!.costUsd = 0.0123; + const csv = toCsv(c); + assert.ok(csv.includes(",0.0123,")); + }); }); // ---------------------------------------------------------------------------- @@ -369,6 +424,7 @@ describe("loadExistingResults", () => { test("round-trips toCsv output back into TestResult objects", async () => { const c = makeComparison(); + c.runs[0]!.results[0]!.costUsd = 0.05; const csv = toCsv(c); // Write to a temp file @@ -392,6 +448,7 @@ describe("loadExistingResults", () => { assert.equal(caseA.passCount, 1); assert.equal(caseA.failCount, 1); assert.equal(caseA.durationMs, 1200); + assert.equal(caseA.costUsd, 0.05); // Check opencode model case-b const ocResults = byModel.get("opencode/llama-qwen7b/qwen2.5-coder-7b"); diff --git a/src/tests/eval-history.test.ts b/src/tests/eval-history.test.ts new file mode 100644 index 0000000..19326a5 --- /dev/null +++ b/src/tests/eval-history.test.ts @@ -0,0 +1,250 @@ +// ============================================================================ +// EVAL HISTORY — unit tests +// ============================================================================ +// Tests for src/eval/history.ts: appendHistory/loadHistory persistence and +// buildTrends grouping, strict-vs-all filtering, and regime-change markers. + +import { test, describe, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + toHistoryEntries, + appendHistory, + loadHistory, + buildTrends, +} from "../eval/history.js"; +import type { EvalComparison, RunProvenance } from "../eval/types.js"; + +const _cleanupDirs: string[] = []; +afterEach(() => { + for (const d of _cleanupDirs.splice(0)) + rmSync(d, { recursive: true, force: true }); +}); + +function tmpDir(): string { + const dir = join( + tmpdir(), + `eval-history-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ); + mkdirSync(dir, { recursive: true }); + _cleanupDirs.push(dir); + return dir; +} + +function makeProvenance(overrides: Partial = {}): RunProvenance { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "sha1", + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "prompt-hash-1", + evalHash: "eval-hash-1", + comparisonFingerprint: "fingerprint-1", + ...overrides, + }; +} + +function makeComparison( + overrides: Partial = {}, +): EvalComparison { + const model = { provider: "claude" as const, model: "sonnet" }; + return { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + models: [model], + runs: [ + { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + model, + results: [ + { + caseId: "case-1", + output: "out", + criteria: [{ criterion: "C1", pass: true, reason: "ok" }], + passCount: 1, + failCount: 0, + durationMs: 500, + costUsd: 0.01, + }, + ], + totalPass: 1, + totalCriteria: 1, + totalCostUsd: 0.01, + }, + ], + comparisonTable: [], + provenance: makeProvenance(), + ...overrides, + }; +} + +describe("toHistoryEntries", () => { + test("produces one entry per model run, carrying provenance", () => { + const c = makeComparison(); + const entries = toHistoryEntries(c); + assert.equal(entries.length, 1); + assert.equal(entries[0]!.evalName, "sample-eval"); + assert.equal(entries[0]!.modelLabel, "claude/sonnet"); + assert.equal(entries[0]!.passCount, 1); + assert.equal(entries[0]!.totalCriteria, 1); + assert.equal(entries[0]!.pct, 1); + assert.equal(entries[0]!.costUsd, 0.01); + assert.equal(entries[0]!.durationMs, 500); + assert.equal(entries[0]!.gitSha, "sha1"); + assert.equal(entries[0]!.comparisonFingerprint, "fingerprint-1"); + }); +}); + +describe("appendHistory / loadHistory", () => { + test("round-trips entries through a JSONL file", () => { + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory(makeComparison(), historyPath); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 1); + assert.equal(loaded[0]!.evalName, "sample-eval"); + }); + + test("appends across multiple calls rather than overwriting", () => { + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory( + makeComparison({ + provenance: makeProvenance({ runAt: "2026-01-01T00:00:00.000Z" }), + }), + historyPath, + ); + appendHistory( + makeComparison({ + provenance: makeProvenance({ runAt: "2026-01-02T00:00:00.000Z" }), + }), + historyPath, + ); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 2); + }); + + test("loadHistory returns [] when the file doesn't exist", () => { + assert.deepEqual(loadHistory("/nonexistent/history.jsonl"), []); + }); + + test("creates parent directories as needed", () => { + const dir = tmpDir(); + const historyPath = join(dir, "nested", "dir", "history.jsonl"); + appendHistory(makeComparison(), historyPath); + assert.equal(loadHistory(historyPath).length, 1); + }); +}); + +describe("buildTrends", () => { + test("groups entries by eval+model and sorts by runAt", () => { + const entries = [ + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-03T00:00:00.000Z", + }, + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-01T00:00:00.000Z", + }, + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-02T00:00:00.000Z", + }, + ]; + const groups = buildTrends(entries, "all"); + assert.equal(groups.length, 1); + assert.deepEqual( + groups[0]!.points.map((p) => p.runAt), + [ + "2026-01-01T00:00:00.000Z", + "2026-01-02T00:00:00.000Z", + "2026-01-03T00:00:00.000Z", + ], + ); + }); + + test("separates different evals and models into distinct groups", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + base, + { ...base, evalName: "other-eval" }, + { ...base, modelLabel: "claude/opus" }, + ]; + const groups = buildTrends(entries, "all"); + assert.equal(groups.length, 3); + }); + + test("all mode keeps every run and flags regime-change points", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { + ...base, + runAt: "2026-01-01T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-02T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-03T00:00:00.000Z", + comparisonFingerprint: "fp-b", + }, + ]; + const [group] = buildTrends(entries, "all"); + assert.equal(group!.points.length, 3); + assert.deepEqual( + group!.points.map((p) => p.regimeChange), + [false, false, true], + ); + }); + + test("strict mode keeps only runs matching the latest fingerprint", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { + ...base, + runAt: "2026-01-01T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-02T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-03T00:00:00.000Z", + comparisonFingerprint: "fp-b", + }, + ]; + const [group] = buildTrends(entries, "strict"); + assert.equal(group!.points.length, 1); + assert.equal(group!.points[0]!.runAt, "2026-01-03T00:00:00.000Z"); + assert.equal(group!.points[0]!.regimeChange, false); + }); + + test("returns groups sorted by evalName then modelLabel", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { ...base, evalName: "zeta-eval" }, + { ...base, evalName: "alpha-eval" }, + ]; + const groups = buildTrends(entries, "all"); + assert.deepEqual( + groups.map((g) => g.evalName), + ["alpha-eval", "zeta-eval"], + ); + }); +}); diff --git a/src/tests/eval-provenance.test.ts b/src/tests/eval-provenance.test.ts new file mode 100644 index 0000000..db11991 --- /dev/null +++ b/src/tests/eval-provenance.test.ts @@ -0,0 +1,167 @@ +// ============================================================================ +// EVAL PROVENANCE — unit tests +// ============================================================================ +// Tests for src/eval/provenance.ts: git SHA/repo capture, judge version +// lookup, hashing, and the composed comparisonFingerprint. + +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; + +import { + getGitSha, + getRepoSlug, + hashContent, + getEvalHash, + computeComparisonFingerprint, + buildProvenance, +} from "../eval/provenance.js"; +import type { EvalFile } from "../eval/types.js"; + +function makeEvalFile(overrides: Partial = {}): EvalFile { + return { + name: "sample-eval", + prompt: "/fake/prompt.txt", + placeholders: ["A"], + testCases: [ + { id: "case-1", vars: { A: "hello" }, criteria: ["Is non-empty"] }, + ], + ...overrides, + }; +} + +describe("hashContent", () => { + test("is deterministic for the same input", () => { + assert.equal(hashContent("hello world"), hashContent("hello world")); + }); + + test("differs for different input", () => { + assert.notEqual(hashContent("a"), hashContent("b")); + }); + + test("returns a 12-character hex string", () => { + const h = hashContent("anything"); + assert.equal(h.length, 12); + assert.ok(/^[0-9a-f]{12}$/.test(h)); + }); +}); + +describe("getEvalHash", () => { + test("is deterministic for the same eval file", () => { + const f = makeEvalFile(); + assert.equal(getEvalHash(f), getEvalHash(f)); + }); + + test("is insensitive to test case ordering", () => { + const a = makeEvalFile({ + testCases: [ + { id: "case-1", vars: {}, criteria: ["C1"] }, + { id: "case-2", vars: {}, criteria: ["C2"] }, + ], + }); + const b = makeEvalFile({ + testCases: [ + { id: "case-2", vars: {}, criteria: ["C2"] }, + { id: "case-1", vars: {}, criteria: ["C1"] }, + ], + }); + assert.equal(getEvalHash(a), getEvalHash(b)); + }); + + test("changes when a criterion changes", () => { + const a = makeEvalFile(); + const b = makeEvalFile({ + testCases: [ + { + id: "case-1", + vars: { A: "hello" }, + criteria: ["Different criterion"], + }, + ], + }); + assert.notEqual(getEvalHash(a), getEvalHash(b)); + }); + + test("changes when a fixture var value changes", () => { + const a = makeEvalFile(); + const b = makeEvalFile({ + testCases: [ + { id: "case-1", vars: { A: "goodbye" }, criteria: ["Is non-empty"] }, + ], + }); + assert.notEqual(getEvalHash(a), getEvalHash(b)); + }); +}); + +describe("computeComparisonFingerprint", () => { + test("is deterministic", () => { + const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + assert.equal(fp1, fp2); + }); + + test("changes when the judge model changes", () => { + const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + const fp2 = computeComparisonFingerprint("claude", "opus", "ph1", "eh1"); + assert.notEqual(fp1, fp2); + }); + + test("changes when the judge prompt hash changes", () => { + const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph2", "eh1"); + assert.notEqual(fp1, fp2); + }); + + test("changes when the eval hash changes", () => { + const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh2"); + assert.notEqual(fp1, fp2); + }); +}); + +describe("getGitSha / getRepoSlug", () => { + test("getGitSha returns a 40-char hex SHA when run inside a git repo", () => { + const sha = getGitSha(); + assert.ok(sha === undefined || /^[0-9a-f]{40}$/.test(sha)); + }); + + test("getRepoSlug returns an owner/repo string or undefined", () => { + const slug = getRepoSlug(); + assert.ok(slug === undefined || /^[^/]+\/[^/]+$/.test(slug)); + }); +}); + +describe("buildProvenance", () => { + test("returns a fully-formed RunProvenance record", () => { + const p = buildProvenance(makeEvalFile()); + assert.equal(p.judgeProvider, "claude"); + assert.ok(typeof p.judgeModel === "string" && p.judgeModel.length > 0); + assert.ok(!Number.isNaN(Date.parse(p.runAt))); + assert.ok(/^[0-9a-f]{12}$/.test(p.judgePromptHash)); + assert.ok(/^[0-9a-f]{12}$/.test(p.evalHash)); + assert.ok(/^[0-9a-f]{12}$/.test(p.comparisonFingerprint)); + }); + + test("comparisonFingerprint matches recomputing from the record's own fields", () => { + const p = buildProvenance(makeEvalFile()); + const recomputed = computeComparisonFingerprint( + p.judgeProvider, + p.judgeModel, + p.judgePromptHash, + p.evalHash, + ); + assert.equal(p.comparisonFingerprint, recomputed); + }); + + test("evalHash differs between two eval files with different criteria", () => { + const p1 = buildProvenance(makeEvalFile()); + const p2 = buildProvenance( + makeEvalFile({ + testCases: [ + { id: "case-1", vars: { A: "hello" }, criteria: ["A different one"] }, + ], + }), + ); + assert.notEqual(p1.evalHash, p2.evalHash); + assert.notEqual(p1.comparisonFingerprint, p2.comparisonFingerprint); + }); +}); diff --git a/src/tests/eval-trend.test.ts b/src/tests/eval-trend.test.ts new file mode 100644 index 0000000..d008926 --- /dev/null +++ b/src/tests/eval-trend.test.ts @@ -0,0 +1,102 @@ +// ============================================================================ +// EVAL:TREND — unit tests +// ============================================================================ +// Tests for src/eval/trend-index.ts CLI arg parsing and the printTrends +// terminal renderer (report.ts). + +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; + +import { parseTrendArgs } from "../eval/trend-index.js"; +import { printTrends } from "../eval/report.js"; +import { buildTrends, toHistoryEntries } from "../eval/history.js"; +import type { EvalComparison } from "../eval/types.js"; + +describe("parseTrendArgs", () => { + test("defaults to results/eval-history.jsonl and all mode", () => { + const args = parseTrendArgs([]); + assert.equal(args.historyPath, "results/eval-history.jsonl"); + assert.equal(args.mode, "all"); + assert.equal(args.evalFilter, undefined); + }); + + test("--history overrides the default path", () => { + const args = parseTrendArgs(["--history", "custom/path.jsonl"]); + assert.equal(args.historyPath, "custom/path.jsonl"); + }); + + test("--mode strict is parsed", () => { + const args = parseTrendArgs(["--mode", "strict"]); + assert.equal(args.mode, "strict"); + }); + + test("--eval filters to a single eval name", () => { + const args = parseTrendArgs(["--eval", "judge-evaluation"]); + assert.equal(args.evalFilter, "judge-evaluation"); + }); + + test("throws on an invalid --mode value", () => { + assert.throws(() => parseTrendArgs(["--mode", "bogus"]), /Invalid --mode/); + }); +}); + +describe("printTrends", () => { + test("does not throw for an empty group list", () => { + assert.doesNotThrow(() => printTrends([])); + }); + + test("does not throw when rendering groups with a regime change", () => { + const model = { provider: "claude" as const, model: "sonnet" }; + const comparison: EvalComparison = { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + models: [model], + runs: [ + { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + model, + results: [ + { + caseId: "case-1", + output: "out", + criteria: [{ criterion: "C1", pass: true, reason: "ok" }], + passCount: 1, + failCount: 0, + durationMs: 500, + costUsd: 0.01, + }, + ], + totalPass: 1, + totalCriteria: 1, + totalCostUsd: 0.01, + }, + ], + comparisonTable: [], + provenance: { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "a".repeat(40), + judgeProvider: "claude", + judgeModel: "sonnet", + judgePromptHash: "ph1", + evalHash: "eh1", + comparisonFingerprint: "fp1", + }, + }; + const entries = [ + ...toHistoryEntries(comparison), + ...toHistoryEntries({ + ...comparison, + provenance: { + ...comparison.provenance, + runAt: "2026-01-02T00:00:00.000Z", + comparisonFingerprint: "fp2", + }, + }), + ]; + const groups = buildTrends(entries, "all"); + assert.doesNotThrow(() => printTrends(groups)); + assert.equal(groups[0]!.points[1]!.regimeChange, true); + }); +}); diff --git a/src/tests/eval.test.ts b/src/tests/eval.test.ts index 4170121..8d42093 100644 --- a/src/tests/eval.test.ts +++ b/src/tests/eval.test.ts @@ -362,7 +362,8 @@ describe("runPrompt", () => { writeFileSync(templatePath, "Process: {{INPUT}}\n", "utf8"); const result = await runPrompt(templatePath, { INPUT: "test data" }); - assert.equal(result.trim(), "the output text"); + assert.equal(result.output.trim(), "the output text"); + assert.equal(result.costUsd, 0.001); }); test("strips prompt header before substitution", async () => { @@ -699,23 +700,26 @@ test_cases: writeFileSync(counterFile, "0", "utf8"); // Responses (in order of claude invocation): - // Call 0: runPrompt (iter 0 scoring) → text output - // Call 1: judgeOutput (iter 0 scoring) → pass:true (score 1/1, all pass → no refine loop enters) + // Call 0: buildProvenance's "claude --version" lookup (plain text, unparsed) + // Call 1: runPrompt (iter 0 scoring) → text output + // Call 2: judgeOutput (iter 0 scoring) → pass:true (score 1/1, all pass → no refine loop enters) // Since iter 0 all pass, the refine loop is skipped entirely. // // We need the initial run to FAIL so refinement starts. - // Call 0: runPrompt → text output - // Call 1: judgeOutput → pass:false (score 0/1, enters refine loop) - // Call 2: refinePrompt → {template: "Refined template v1"} (saves to disk) - // Call 3: runPrompt (iter 1 re-score) → text output - // Call 4: judgeOutput (iter 1 re-score) → pass:true (score 1/1, new best) - // Call 5: refinePrompt → {template: "Refined template v2"} (saves to disk, but iter 2 regresses) - // Call 6: runPrompt (iter 2 re-score) → text output - // Call 7: judgeOutput (iter 2 re-score) → pass:false (score 0/1, regression) + // Call 1: runPrompt → text output + // Call 2: judgeOutput → pass:false (score 0/1, enters refine loop) + // Call 3: refinePrompt → {template: "Refined template v1"} (saves to disk) + // Call 4: runPrompt (iter 1 re-score) → text output + // Call 5: judgeOutput (iter 1 re-score) → pass:true (score 1/1, new best) + // Call 6: refinePrompt → {template: "Refined template v2"} (saves to disk, but iter 2 regresses) + // Call 7: runPrompt (iter 2 re-score) → text output + // Call 8: judgeOutput (iter 2 re-score) → pass:false (score 0/1, regression) // → max-iter=2 exhausted, best was iter 1 → restore "# Header\n\nRefined template v1\n" const responses = [ - // Call 0: runPrompt initial + // Call 0: buildProvenance's "claude --version" lookup — plain text, never JSON-parsed + "2.0.0-mock (Mock Claude)\n", + // Call 1: runPrompt initial JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: "initial output" }] }, @@ -723,7 +727,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 1: judgeOutput initial → FAIL + // Call 2: judgeOutput initial → FAIL JSON.stringify({ type: "assistant", message: { @@ -738,7 +742,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 2: refinePrompt → template v1 + // Call 3: refinePrompt → template v1 JSON.stringify({ type: "assistant", message: { @@ -753,7 +757,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 3: runPrompt iter 1 re-score + // Call 4: runPrompt iter 1 re-score JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: "iter1 output" }] }, @@ -761,7 +765,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 4: judgeOutput iter 1 → PASS (new best: 1/1) + // Call 5: judgeOutput iter 1 → PASS (new best: 1/1) JSON.stringify({ type: "assistant", message: { @@ -773,7 +777,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 5: refinePrompt → template v2 (but iter 2 will regress) + // Call 6: refinePrompt → template v2 (but iter 2 will regress) JSON.stringify({ type: "assistant", message: { @@ -788,7 +792,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 6: runPrompt iter 2 re-score + // Call 7: runPrompt iter 2 re-score JSON.stringify({ type: "assistant", message: { content: [{ type: "text", text: "iter2 output" }] }, @@ -796,7 +800,7 @@ test_cases: "\n" + JSON.stringify({ type: "result", total_cost_usd: 0 }) + "\n", - // Call 7: judgeOutput iter 2 → FAIL (regression: 0/1) + // Call 8: judgeOutput iter 2 → FAIL (regression: 0/1) JSON.stringify({ type: "assistant", message: { diff --git a/src/tests/statusline-ui.test.ts b/src/tests/statusline-ui.test.ts index e3cd05e..8b37707 100644 --- a/src/tests/statusline-ui.test.ts +++ b/src/tests/statusline-ui.test.ts @@ -96,12 +96,16 @@ describe("App status bar", () => { }); test("names the repo and branch it is running in", async () => { - // The suite runs inside executant's own checkout. + // Asserts a repo-name/branch segment renders before the gauge, without + // hardcoding either value — the checkout directory name and branch vary + // by environment (e.g. a CI runner or sandboxed workspace won't + // necessarily be named after the repo), and a long value legitimately + // shrinks to fit rather than appearing verbatim. await withApp(RUNNING_EVENTS, async ({ lastFrame }) => { - const frame = await waitForFrame(lastFrame, /executant\s+\S+\s+━{10}/, { + const frame = await waitForFrame(lastFrame, /\S+\s+\S+\s+━{10}/, { describe: "the repo and branch segment", }); - assert.match(frame, /executant\s+\S+\s+━{10}/); + assert.match(frame, /\S+\s+\S+\s+━{10}/); }); }); diff --git a/src/tests/statusline.test.ts b/src/tests/statusline.test.ts index bc96150..69b8546 100644 --- a/src/tests/statusline.test.ts +++ b/src/tests/statusline.test.ts @@ -17,6 +17,7 @@ import { buildGauge, contextTokens, contextWindowSize, + fitRepoLabel, readRepoInfo, statusLineEnabled, DEFAULT_CONTEXT_WINDOW, @@ -133,6 +134,51 @@ describe("buildGauge", () => { }); }); +// ---------------------------------------------------------------------------- +// fitRepoLabel +// ---------------------------------------------------------------------------- + +describe("fitRepoLabel", () => { + test("returns the repo unchanged when it already fits", () => { + const repo = { name: "executant", branch: "main" }; + assert.deepEqual(fitRepoLabel(repo, 40), repo); + }); + + test("truncates the name when there is no branch", () => { + assert.deepEqual(fitRepoLabel({ name: "executant" }, 5), { + name: "exec…", + }); + }); + + test("shrinks a long branch while keeping a short name intact", () => { + const repo = { + name: "executant", + branch: "operator/4-enhancement-eval-history-observability-w", + }; + const fitted = fitRepoLabel(repo, 30); + assert.equal(fitted.name, "executant"); + assert.ok(fitted.branch!.length <= 30 - 9 - 3); + assert.ok(fitted.branch!.endsWith("…")); + }); + + test("shrinks both name and branch when both are long", () => { + const repo = { + name: "operator-job-9d914c84-85ec-410e-a24c-2ea43be588a0", + branch: "operator/4-enhancement-eval-history-observability-w", + }; + const fitted = fitRepoLabel(repo, 40); + assert.ok(fitted.name.length + 3 + fitted.branch!.length <= 40); + assert.ok(fitted.name.endsWith("…")); + assert.ok(fitted.branch!.endsWith("…")); + }); + + test("never produces a negative-width slice", () => { + const repo = { name: "a-very-long-repo-name-indeed", branch: "main" }; + const fitted = fitRepoLabel(repo, 1); + assert.equal(fitted.name, "…"); + }); +}); + // ---------------------------------------------------------------------------- // readRepoInfo // ---------------------------------------------------------------------------- diff --git a/src/ui/App.tsx b/src/ui/App.tsx index 6a5a5f1..89a0e35 100644 --- a/src/ui/App.tsx +++ b/src/ui/App.tsx @@ -477,7 +477,11 @@ export function App({ )} {showStatusBar && ( - + )} {isInterjecting diff --git a/src/ui/StatusBar.tsx b/src/ui/StatusBar.tsx index b8fa742..84eb995 100644 --- a/src/ui/StatusBar.tsx +++ b/src/ui/StatusBar.tsx @@ -15,7 +15,9 @@ import { Box, Text } from "ink"; import { buildGauge, contextWindowSize, + fitRepoLabel, readRepoInfo, + GAUGE_WIDTH, type GaugeLevel, type RepoInfo, } from "../lib/statusline.js"; @@ -27,14 +29,21 @@ const LEVEL_COLOR: Record = { high: theme.error, }; +// Everything after the repo/branch segment: 2-space gap + the gauge itself + +// " NNN%" (up to 5 chars) + " NNN.Nk/NNNk" (up to 13 chars, generous for a +// 7-digit token count against a "1M" window). +const GAUGE_SEGMENT_WIDTH = 2 + GAUGE_WIDTH + 5 + 13; + interface Props { /** Context tokens the running session occupies; 0 before its first turn. */ tokens: number; /** The session's model — sizes the gauge (200k, or 1M for `[1m]`). */ model: string; + /** Terminal width, so a long repo name/branch shrinks instead of wrapping onto a second line. */ + columns: number; } -export function StatusBar({ tokens, model }: Props) { +export function StatusBar({ tokens, model, columns }: Props) { const [repo, setRepo] = useState(undefined); useEffect(() => { @@ -51,14 +60,15 @@ export function StatusBar({ tokens, model }: Props) { const gauge = buildGauge(tokens, contextWindowSize(model)); const level = LEVEL_COLOR[gauge.level]; + const fitted = repo && fitRepoLabel(repo, columns - GAUGE_SEGMENT_WIDTH); return ( - {repo && {repo.name}} - {repo?.branch && ( + {fitted && {fitted.name}} + {fitted?.branch && ( {" "} - {repo.branch} + {fitted.branch} )} {" "} From 07add0f5b08384cfe2de323ece22afbe53bc0cb2 Mon Sep 17 00:00:00 2001 From: Coston Perkins Date: Mon, 31 Aug 2026 12:14:32 -0500 Subject: [PATCH 2/5] fix(tests): make CI test runs deterministic (ink CI-mode rendering, zombie-reap race) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three tests failed on GitHub Actions while passing locally: - Ink checks `is-in-ci` at module load and, in CI, defers all frame writes until unmount (and the CI unmount path overwrites the final frame with a bare newline in debug mode), so tests asserting on frames written to a fake stdout captured nothing. A new shared force-non-ci.ts helper — imported first in every ink-rendering test file — pins CI="false", which is-in-ci short-circuits on even when GITHUB_ACTIONS/CI_* vars are set. - The process-group timeout test asserted ESRCH on the first signal-0 probe, but a just-SIGTERMed grandchild can still be an unreaped zombie (signal 0 succeeds on zombies), so it raced on Linux. The assertion now polls up to 2s for the process to disappear. Repro for the ink failures: `CI=true npm test` fails identically on macOS; green after this change both with and without CI=true. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019thBZNhFBYfFP7Dk5cgzMW --- src/tests/app-non-tty-ui.test.ts | 1 + src/tests/app-output-pane-ui.test.ts | 1 + src/tests/command.test.ts | 23 +++++++++++++++++++--- src/tests/force-non-ci.ts | 29 ++++++++++++++++++++++++++++ src/tests/ink-harness.ts | 5 +++++ src/tests/logpane-scroll-ui.test.ts | 1 + src/tests/report-prompt-ui.test.ts | 1 + src/tests/retrospective-ui.test.ts | 1 + src/tests/statusline-ui.test.ts | 1 + 9 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 src/tests/force-non-ci.ts diff --git a/src/tests/app-non-tty-ui.test.ts b/src/tests/app-non-tty-ui.test.ts index 9412c84..2cb7108 100644 --- a/src/tests/app-non-tty-ui.test.ts +++ b/src/tests/app-non-tty-ui.test.ts @@ -23,6 +23,7 @@ // followed by workflow:complete) gets batched by React into too few commits // to ever land on the vulnerable "step running on a non-TTY" render at all. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe } from "node:test"; import assert from "node:assert/strict"; import { EventEmitter } from "node:events"; diff --git a/src/tests/app-output-pane-ui.test.ts b/src/tests/app-output-pane-ui.test.ts index aba563d..24da751 100644 --- a/src/tests/app-output-pane-ui.test.ts +++ b/src/tests/app-output-pane-ui.test.ts @@ -11,6 +11,7 @@ // channel that hook listens on, but asserting exact on-screen row positions // from a text frame is what the pure math tests already cover directly. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe } from "node:test"; import assert from "node:assert/strict"; import React from "react"; diff --git a/src/tests/command.test.ts b/src/tests/command.test.ts index 224c0ad..e0b4e11 100644 --- a/src/tests/command.test.ts +++ b/src/tests/command.test.ts @@ -47,6 +47,20 @@ async function collectEventsExpectingError( } } +/** Polls until signal-0 delivery fails with ESRCH (process fully reaped). */ +async function processGone(pid: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + process.kill(pid, 0); + } catch { + return true; + } + await new Promise((r) => setTimeout(r, 50)); + } + return false; +} + // ---------------------------------------------------------------------------- // runCommand // ---------------------------------------------------------------------------- @@ -158,9 +172,12 @@ describe("runCommand — timeout_seconds", () => { assert.ok(error instanceof TimeoutError); const grandchildPid = Number(readFileSync(pidFile, "utf8").trim()); - assert.throws( - () => process.kill(grandchildPid, 0), - /ESRCH/, + // The group SIGTERM has been sent by the time TimeoutError surfaces, + // but the grandchild can linger briefly as an unreaped zombie — and + // signal 0 still succeeds on a zombie — so poll for it to disappear + // rather than asserting on the very first probe. + assert.ok( + await processGone(grandchildPid, 2000), "the sleep grandchild should have been killed, not left orphaned", ); } finally { diff --git a/src/tests/force-non-ci.ts b/src/tests/force-non-ci.ts new file mode 100644 index 0000000..04a0749 --- /dev/null +++ b/src/tests/force-non-ci.ts @@ -0,0 +1,29 @@ +// ============================================================================ +// FORCE NON-CI ENVIRONMENT FOR INK RENDERING TESTS +// ============================================================================ +// Ink checks the `is-in-ci` package once at module load, and when it detects a +// CI environment it defers all frame rendering: `onRender` stashes each frame +// in memory instead of writing it, and only the final frame is written at +// unmount. On top of that, the CI unmount path writes `lastOutput + "\n"` even +// in debug mode (where `lastOutput` is never populated), so a component that +// exits on its own overwrites its real final frame with a bare "\n". Either +// way, a test that asserts on frames written to a fake stdout sees nothing — +// which is exactly what happened on GitHub Actions (CI=true) while the same +// tests passed locally. +// +// `is-in-ci` computes, at module evaluation time: +// +// env.CI !== '0' && env.CI !== 'false' +// && ('CI' in env || 'CONTINUOUS_INTEGRATION' in env +// || Object.keys(env).some(key => key.startsWith('CI_'))) +// +// Setting CI="false" fails the first conjunct, which short-circuits the whole +// expression — so this guarantees "not CI" even on a real Actions runner where +// GITHUB_ACTIONS=true and CI_* variables are also set. +// +// IMPORTANT: this module must be the FIRST import of any test file that +// renders Ink (directly, via ink-testing-library, or via a `../ui/*` +// component, all of which import ink) — `is-in-ci` reads the env when its +// module evaluates, and ESM evaluates imports in listed order. + +process.env["CI"] = "false"; diff --git a/src/tests/ink-harness.ts b/src/tests/ink-harness.ts index 02875b5..99aca5b 100644 --- a/src/tests/ink-harness.ts +++ b/src/tests/ink-harness.ts @@ -14,6 +14,11 @@ // missed its deadline and the run wedged for as long as it was allowed to. // // withInk() makes the unmount unconditional, so a failure stays a failure. +// +// NOTE: every test file that renders Ink must have +// `import "./force-non-ci.js";` as its FIRST import (importing this harness is +// not enough — a `../ui/*` import listed earlier would load ink first). See +// force-non-ci.ts for why Ink's CI detection breaks frame assertions. import { render } from "ink-testing-library"; import type { ReactElement } from "react"; diff --git a/src/tests/logpane-scroll-ui.test.ts b/src/tests/logpane-scroll-ui.test.ts index a9e48d4..990f388 100644 --- a/src/tests/logpane-scroll-ui.test.ts +++ b/src/tests/logpane-scroll-ui.test.ts @@ -5,6 +5,7 @@ // with ink-testing-library rather than going through the full App — no event // stream or reducer needed to verify the scroll-offset windowing math. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import assert from "node:assert/strict"; import { describe, test } from "node:test"; import React from "react"; diff --git a/src/tests/report-prompt-ui.test.ts b/src/tests/report-prompt-ui.test.ts index d2d777e..a55e2f5 100644 --- a/src/tests/report-prompt-ui.test.ts +++ b/src/tests/report-prompt-ui.test.ts @@ -10,6 +10,7 @@ // ink-testing-library's stdin reports isTTY and implements setRawMode, so // Ink's useInput is live and `stdin.write("a")` behaves like a real keypress. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; import React from "react"; diff --git a/src/tests/retrospective-ui.test.ts b/src/tests/retrospective-ui.test.ts index 769ce9c..20240c2 100644 --- a/src/tests/retrospective-ui.test.ts +++ b/src/tests/retrospective-ui.test.ts @@ -11,6 +11,7 @@ // ink-testing-library's stdin reports isTTY and implements setRawMode, so // Ink's useInput is live and `stdin.write("u")` behaves like a real keypress. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; import React from "react"; diff --git a/src/tests/statusline-ui.test.ts b/src/tests/statusline-ui.test.ts index 8b37707..98a8b17 100644 --- a/src/tests/statusline-ui.test.ts +++ b/src/tests/statusline-ui.test.ts @@ -5,6 +5,7 @@ // it starts empty, moves as per-call output:context events land, ignores the // cumulative output:usage totals, and disappears under EXECUTANT_STATUSLINE=0. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe, beforeEach, afterEach } from "node:test"; import assert from "node:assert/strict"; import React from "react"; From 13439a5fbf8f3254628b07701b850755762be427 Mon Sep 17 00:00:00 2001 From: Coston Perkins Date: Mon, 31 Aug 2026 12:37:53 -0500 Subject: [PATCH 3/5] fix(eval): make history provenance truthful at the edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An observability feature must never produce records that are confidently wrong. Four ways it could, found by running it, now fixed: - A resumed run (cached --output-csv results) appended a history record stamped with fresh provenance — new runAt/gitSha/fingerprint — for scores produced under the previous run's regime. recordHistory now refuses to append when any case was reused, and says why. - The judge model was asserted, not observed: provenance recorded EXECUTANT_MODEL ?? sonnet while the judge ran with no --model flag at all, i.e. the user's CLI default. judge.ts now pins the model it runs to the exact value provenance records (resolveJudgeModel, shared). Likewise the bare single-model path hard-coded the "claude/sonnet" series label; it now records the model that actually ran. - judgeVersion was captured but excluded from comparisonFingerprint, though the docs and the regime-marker text claim version changes are flagged — a Claude CLI upgrade shifted judging silently. The version is now part of the fingerprint. - One corrupt history line (a half-written append) aborted eval:trend entirely; it is now skipped with a warning naming the file and line. Plus smaller honesty fixes: judgePromptHash now hashes the header-stripped prompt (what the judge actually receives), evalHash is insensitive to YAML var-key order, trend flags missing a value now error instead of silently falling back (a silent --mode strict miss hands the user non-comparable data), an --eval filter that matches nothing says so instead of claiming no history exists, and trend group identity no longer round-trips through the "::" map key. Every fix carries a regression test (9 new). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019thBZNhFBYfFP7Dk5cgzMW --- docs/eval-comparison.md | 13 ++- src/eval/history.ts | 27 +++++-- src/eval/index.ts | 59 +++++++++----- src/eval/judge.ts | 6 ++ src/eval/provenance.ts | 50 +++++++++--- src/eval/trend-index.ts | 37 +++++++-- src/eval/types.ts | 7 ++ src/tests/eval-history.test.ts | 32 +++++++- src/tests/eval-provenance.test.ts | 129 ++++++++++++++++++++++++++++-- src/tests/eval-trend.test.ts | 11 +++ src/tests/eval.test.ts | 81 +++++++++++++++++++ 11 files changed, 399 insertions(+), 53 deletions(-) diff --git a/docs/eval-comparison.md b/docs/eval-comparison.md index 3aa5ed5..891d715 100644 --- a/docs/eval-comparison.md +++ b/docs/eval-comparison.md @@ -74,11 +74,11 @@ change in the judge/eval regime itself, and the two should never be confused: | `runAt` | ISO timestamp of the run | | `repo` | `owner/repo`, parsed from the `origin` git remote (GitHub only) | | `gitSha` | Commit evaluated (`git rev-parse HEAD`) | -| `judgeProvider` / `judgeModel` | The judge is always Claude — this records which model | +| `judgeProvider` / `judgeModel` | The judge is always Claude — this records which model. The judge is pinned to `EXECUTANT_MODEL` (default `sonnet`), never the CLI's configured default, so the recorded model is the one that actually judged | | `judgeVersion` | `claude --version`, when it can be read | -| `judgePromptHash` | Hash of `src/eval/prompts/criterion-judge.txt` | +| `judgePromptHash` | Hash of `src/eval/prompts/criterion-judge.txt` (header-stripped — the text the judge actually receives) | | `evalHash` | Hash of the resolved eval spec (test cases, vars, criteria) | -| `comparisonFingerprint` | Hash of judge provider+model+prompt hash+eval hash — the strict-comparability key | +| `comparisonFingerprint` | Hash of judge provider+model+version+prompt hash+eval hash — the strict-comparability key | Each case's API cost (USD, Claude only — OpenCode/local models don't report cost) is captured alongside its score and duration on every `TestResult`, and @@ -217,7 +217,12 @@ Two modes: under a different judge/prompt/eval config. Each trend point can always be traced back to its `run_at`, `git_sha`, and -full judge config via the underlying history record. +full judge config via the underlying history record. To keep that guarantee, +a run that reuses any cached case result from an existing `--output-csv` +skips the history append entirely (the cached scores were produced under the +previous run's provenance) — delete the CSV to re-run and record. A corrupt +line in the history file (e.g. from an interrupted append) is skipped with a +warning rather than aborting the report. ## Adding a new model diff --git a/src/eval/history.ts b/src/eval/history.ts index b27bda2..50b4a2e 100644 --- a/src/eval/history.ts +++ b/src/eval/history.ts @@ -74,13 +74,27 @@ export function appendHistory( appendFileSync(historyPath, lines, "utf8"); } -/** Reads all history records from a JSONL file. Returns [] if the file doesn't exist. */ +/** + * Reads all history records from a JSONL file. Returns [] if the file doesn't + * exist. A line that fails to parse (e.g. a half-written trailing line from an + * interrupted append) is skipped with a warning naming the file and line — + * one corrupt line must never make months of accumulated history unreadable. + */ export function loadHistory(historyPath: string): HistoryEntry[] { if (!existsSync(historyPath)) return []; return readFileSync(historyPath, "utf8") .split("\n") - .filter((line) => line.trim()) - .map((line) => JSON.parse(line) as HistoryEntry); + .flatMap((line, i) => { + if (!line.trim()) return []; + try { + return [JSON.parse(line) as HistoryEntry]; + } catch (err) { + process.stderr.write( + `[eval:trend] warning: skipping corrupt line ${i + 1} of ${historyPath}: ${err instanceof Error ? err.message : String(err)}\n`, + ); + return []; + } + }); } export type TrendMode = "strict" | "all"; @@ -115,8 +129,11 @@ export function buildTrends( } const groups: TrendGroup[] = []; - for (const [key, groupEntries] of byGroup) { - const [evalName, label] = key.split("::") as [string, string]; + for (const groupEntries of byGroup.values()) { + // Identity comes from the entries themselves, not from re-splitting the + // map key — a label that ever contained the delimiter would round-trip + // truncated. + const { evalName, modelLabel: label } = groupEntries[0]!; const sorted = [...groupEntries].sort((a, b) => a.runAt.localeCompare(b.runAt), ); diff --git a/src/eval/index.ts b/src/eval/index.ts index 411faaa..13cfed6 100644 --- a/src/eval/index.ts +++ b/src/eval/index.ts @@ -29,6 +29,7 @@ import { printDiff, } from "./report.js"; import { toJson, toCsv, modelLabel } from "./export.js"; +import { DEFAULT_MODEL } from "../lib/utils.js"; import { buildProvenance } from "./provenance.js"; import { appendHistory } from "./history.js"; import type { @@ -282,12 +283,14 @@ async function runEval( ? applyCaseFilter(evalFile.testCases, caseFilter) : evalFile.testCases; const results: TestResult[] = []; + let cachedCount = 0; for (const tc of cases) { const hit = cached?.get(tc.id); if (hit) { process.stdout.write(` skipping ${tc.id} (cached)\n`); results.push(hit); + cachedCount++; continue; } process.stdout.write(` running ${tc.id}…`); @@ -346,9 +349,33 @@ async function runEval( totalPass, totalCriteria, totalCostUsd, + cachedCount, }; } +/** + * Appends to the history log only when every result in the comparison was + * freshly executed. A resumed run reuses scores produced under the previous + * run's provenance (different time, commit, possibly a different judge), so + * appending them under today's provenance would fabricate a trend point. + */ +export function recordHistory( + comparison: EvalComparison, + historyPath: string, +): void { + const cachedTotal = comparison.runs.reduce( + (sum, run) => sum + (run.cachedCount ?? 0), + 0, + ); + if (cachedTotal > 0) { + console.log( + ` Skipping history append: ${cachedTotal} case result(s) were reused from the output CSV, so this run's provenance does not describe them. Delete the CSV to re-run and record.`, + ); + return; + } + appendHistory(comparison, historyPath); +} + export function collectFailures( run: EvalRun, evalFile: ReturnType, @@ -505,29 +532,31 @@ async function runEvalFile( if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); - if (args.historyPath) appendHistory(comparison, args.historyPath); + if (args.historyPath) recordHistory(comparison, args.historyPath); return; } - // Single-model mode — load cached results for resume support - const singleModel = args.models[0]; + // Single-model mode — load cached results for resume support. When no + // --models was given, resolve the same default runPrompt will actually use + // (EXECUTANT_MODEL, then sonnet) and pass it explicitly, so the recorded + // model label always names the model that ran — previously + // `EXECUTANT_MODEL=haiku` runs were filed under "claude/sonnet". + const model = args.models[0] ?? { + provider: "claude" as const, + model: process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL, + }; const existing = outputCsv ? loadExistingResults(outputCsv) : new Map(); - const label = singleModel ? modelLabel(singleModel) : "claude/sonnet"; let run = await runEval( evalFile, undefined, - singleModel, - existing.get(label), + model, + existing.get(modelLabel(model)), args.caseFilter, ); printRun(run); // Write output files (wraps single-model run in a minimal comparison) if (outputJson || outputCsv || args.historyPath) { - const model = singleModel ?? { - provider: "claude" as const, - model: "sonnet", - }; const comparison: EvalComparison = { evalName: evalFile.name, templatePath: evalFile.prompt, @@ -538,7 +567,7 @@ async function runEvalFile( }; if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); - if (args.historyPath) appendHistory(comparison, args.historyPath); + if (args.historyPath) recordHistory(comparison, args.historyPath); } if (!args.refine || run.totalPass === run.totalCriteria) return; @@ -556,13 +585,7 @@ async function runEvalFile( saveRefinedTemplate(evalFile.prompt, improved); printRefinementHeader(iter, args.maxIter); - run = await runEval( - evalFile, - undefined, - singleModel, - undefined, - args.caseFilter, - ); + run = await runEval(evalFile, undefined, model, undefined, args.caseFilter); printRun(run); if (run.totalPass > bestRun.totalPass) { diff --git a/src/eval/judge.ts b/src/eval/judge.ts index cd49300..7ae33f5 100644 --- a/src/eval/judge.ts +++ b/src/eval/judge.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import { z } from "zod"; import { runClaudeStructured } from "../tasks/claude.js"; import { stripPromptHeader } from "../lib/utils.js"; +import { resolveJudgeModel } from "./provenance.js"; import type { CriterionResult } from "./types.js"; const __dir = dirname(fileURLToPath(import.meta.url)); @@ -40,6 +41,11 @@ export async function judgeOutput( type: "claude", name: "eval:criterion-judge", prompt, + // Pinned so the judge that runs is the judge provenance records — + // without this the CLI falls back to the user's configured default + // model, and two machines with different defaults would share a + // "strictly comparable" fingerprint for genuinely different judges. + model: resolveJudgeModel(), allowedTools: [], permissionMode: "default", }, diff --git a/src/eval/provenance.ts b/src/eval/provenance.ts index 09e6c52..7ae02ef 100644 --- a/src/eval/provenance.ts +++ b/src/eval/provenance.ts @@ -11,9 +11,18 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { DEFAULT_MODEL } from "../lib/utils.js"; +import { DEFAULT_MODEL, stripPromptHeader } from "../lib/utils.js"; import type { EvalFile, RunProvenance } from "./types.js"; +/** + * The model the criterion judge runs with. Shared with judge.ts, which pins + * this model on its Claude calls — provenance must record the judge that + * actually ran, not an assumption about it. + */ +export function resolveJudgeModel(): string { + return process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL; +} + const __dir = dirname(fileURLToPath(import.meta.url)); function tryExec(cmd: string): string | undefined { @@ -60,10 +69,21 @@ export function hashContent(content: string): string { return createHash("sha256").update(content).digest("hex").slice(0, 12); } -/** Hash of the judge prompt template — changes here signal a judging-regime change. */ -function getJudgePromptHash(): string { +/** + * Hash of the judge prompt template — changes here signal a judging-regime + * change. Hashes the header-stripped text (what judge.ts actually sends), so + * editing only the documentation header does not flag a false regime change. + */ +export function getJudgePromptHash(): string { const promptPath = join(__dir, "prompts", "criterion-judge.txt"); - return hashContent(readFileSync(promptPath, "utf8")); + return hashContent(stripPromptHeader(readFileSync(promptPath, "utf8"))); +} + +/** Sorts an object's keys so semantically-equal vars hash identically regardless of YAML key order. */ +function sortedVars(vars: Record): Record { + return Object.fromEntries( + Object.entries(vars).sort(([a], [b]) => a.localeCompare(b)), + ); } /** Hash of the resolved eval spec (test cases + criteria) — changes here signal an eval-regime change. */ @@ -73,27 +93,38 @@ export function getEvalHash(evalFile: EvalFile): string { placeholders: [...evalFile.placeholders].sort(), testCases: [...evalFile.testCases] .sort((a, b) => a.id.localeCompare(b.id)) - .map((tc) => ({ id: tc.id, vars: tc.vars, criteria: tc.criteria })), + .map((tc) => ({ + id: tc.id, + vars: sortedVars(tc.vars), + criteria: tc.criteria, + })), }); return hashContent(canonical); } -/** Stable fingerprint of judge+prompt+eval config — the strict-comparability key. */ +/** + * Stable fingerprint of judge+prompt+eval config — the strict-comparability + * key. Includes the judge CLI version: a CLI upgrade can change judging + * behavior (new default snapshot, new system prompt), so runs across an + * upgrade are a regime change, not a comparable series. + */ export function computeComparisonFingerprint( judgeProvider: string, judgeModel: string, + judgeVersion: string | undefined, judgePromptHash: string, evalHash: string, ): string { return hashContent( - `${judgeProvider}:${judgeModel}:${judgePromptHash}:${evalHash}`, + `${judgeProvider}:${judgeModel}:${judgeVersion ?? "unknown"}:${judgePromptHash}:${evalHash}`, ); } /** Builds the full provenance record for a comparison run over the given eval file. */ export function buildProvenance(evalFile: EvalFile): RunProvenance { const judgeProvider = "claude"; - const judgeModel = process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL; + const judgeModel = resolveJudgeModel(); + const judgeVersion = getJudgeVersion(); const judgePromptHash = getJudgePromptHash(); const evalHash = getEvalHash(evalFile); return { @@ -102,12 +133,13 @@ export function buildProvenance(evalFile: EvalFile): RunProvenance { gitSha: getGitSha(), judgeProvider, judgeModel, - judgeVersion: getJudgeVersion(), + judgeVersion, judgePromptHash, evalHash, comparisonFingerprint: computeComparisonFingerprint( judgeProvider, judgeModel, + judgeVersion, judgePromptHash, evalHash, ), diff --git a/src/eval/trend-index.ts b/src/eval/trend-index.ts index a652c58..9c74397 100644 --- a/src/eval/trend-index.ts +++ b/src/eval/trend-index.ts @@ -25,6 +25,20 @@ interface TrendArgs { evalFilter?: string; } +/** + * Returns the value at `i`, or throws when the flag has none (end of args, or + * the next token is another flag). A silently-ignored `--mode strict` would + * hand the user non-comparable data while they believe it's strict — the one + * silent failure this tool exists to prevent. + */ +function takeValue(rawArgs: string[], i: number, flag: string): string { + const value = rawArgs[i]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`Missing value for ${flag}`); + } + return value; +} + export function parseTrendArgs(rawArgs: string[]): TrendArgs { let historyPath = DEFAULT_HISTORY_PATH; let mode: TrendMode = "all"; @@ -45,18 +59,18 @@ export function parseTrendArgs(rawArgs: string[]): TrendArgs { ].join("\n"), ); process.exit(0); - } else if (arg === "--history" && rawArgs[i + 1]) { - historyPath = rawArgs[++i]!; - } else if (arg === "--mode" && rawArgs[i + 1]) { - const value = rawArgs[++i]!; + } else if (arg === "--history") { + historyPath = takeValue(rawArgs, ++i, "--history"); + } else if (arg === "--mode") { + const value = takeValue(rawArgs, ++i, "--mode"); if (value !== "strict" && value !== "all") { throw new Error( `Invalid --mode "${value}": expected "strict" or "all"`, ); } mode = value; - } else if (arg === "--eval" && rawArgs[i + 1]) { - evalFilter = rawArgs[++i]; + } else if (arg === "--eval") { + evalFilter = takeValue(rawArgs, ++i, "--eval"); } } @@ -65,16 +79,23 @@ export function parseTrendArgs(rawArgs: string[]): TrendArgs { export async function main(): Promise { const args = parseTrendArgs(process.argv.slice(2)); - const entries = loadHistory(args.historyPath).filter( + const allEntries = loadHistory(args.historyPath); + const entries = allEntries.filter( (e) => !args.evalFilter || e.evalName === args.evalFilter, ); - if (entries.length === 0) { + if (allEntries.length === 0) { console.log( `No history found at ${args.historyPath}. Run evals with "--history ${args.historyPath}" to start tracking trends.`, ); return; } + if (entries.length === 0) { + console.log( + `No records match --eval "${args.evalFilter}" in ${args.historyPath} (${allEntries.length} record(s) for: ${[...new Set(allEntries.map((e) => e.evalName))].join(", ")}).`, + ); + return; + } const groups = buildTrends(entries, args.mode); console.log( diff --git a/src/eval/types.ts b/src/eval/types.ts index 8fd7b02..d58fd56 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -36,6 +36,13 @@ export interface EvalRun { totalCriteria: number; /** Sum of results[].costUsd. Undefined when no result reported a cost. */ totalCostUsd?: number; + /** + * How many results came from a resumed --output-csv rather than running. + * Non-zero blocks history appends: a cached score was produced under the + * *old* run's provenance, so stamping it with a fresh one would fabricate + * a trend point. + */ + cachedCount?: number; } export interface FailureContext { diff --git a/src/tests/eval-history.test.ts b/src/tests/eval-history.test.ts index 19326a5..5912d00 100644 --- a/src/tests/eval-history.test.ts +++ b/src/tests/eval-history.test.ts @@ -6,7 +6,7 @@ import { test, describe, afterEach } from "node:test"; import assert from "node:assert/strict"; -import { mkdirSync, rmSync } from "node:fs"; +import { appendFileSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -16,6 +16,7 @@ import { loadHistory, buildTrends, } from "../eval/history.js"; +import { recordHistory } from "../eval/index.js"; import type { EvalComparison, RunProvenance } from "../eval/types.js"; const _cleanupDirs: string[] = []; @@ -142,6 +143,35 @@ describe("appendHistory / loadHistory", () => { appendHistory(makeComparison(), historyPath); assert.equal(loadHistory(historyPath).length, 1); }); + + test("recordHistory refuses to append when any result was resumed from a CSV", () => { + // A cached score was produced under the *previous* run's provenance; + // stamping it with today's runAt/gitSha/fingerprint fabricates a trend + // point that never happened. + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + + const cached = makeComparison(); + cached.runs = [{ ...cached.runs[0]!, cachedCount: 1 }]; + recordHistory(cached, historyPath); + assert.deepEqual(loadHistory(historyPath), []); + + recordHistory(makeComparison(), historyPath); + assert.equal(loadHistory(historyPath).length, 1); + }); + + test("skips a corrupt line instead of losing the whole log", () => { + // A half-written trailing line is what an interrupted append leaves + // behind; it must cost one warning, not the entire history. + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory(makeComparison(), historyPath); + appendFileSync(historyPath, '{"runAt":"2026-01-02T00:0', "utf8"); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 1); + assert.equal(loaded[0]!.evalName, "sample-eval"); + }); }); describe("buildTrends", () => { diff --git a/src/tests/eval-provenance.test.ts b/src/tests/eval-provenance.test.ts index db11991..7790024 100644 --- a/src/tests/eval-provenance.test.ts +++ b/src/tests/eval-provenance.test.ts @@ -7,14 +7,19 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + import { getGitSha, getRepoSlug, hashContent, getEvalHash, + getJudgePromptHash, computeComparisonFingerprint, buildProvenance, } from "../eval/provenance.js"; +import { stripPromptHeader } from "../lib/utils.js"; import type { EvalFile } from "../eval/types.js"; function makeEvalFile(overrides: Partial = {}): EvalFile { @@ -67,6 +72,18 @@ describe("getEvalHash", () => { assert.equal(getEvalHash(a), getEvalHash(b)); }); + test("is insensitive to var key ordering within a test case", () => { + const a = makeEvalFile({ + placeholders: ["A", "B"], + testCases: [{ id: "case-1", vars: { A: "x", B: "y" }, criteria: ["C1"] }], + }); + const b = makeEvalFile({ + placeholders: ["A", "B"], + testCases: [{ id: "case-1", vars: { B: "y", A: "x" }, criteria: ["C1"] }], + }); + assert.equal(getEvalHash(a), getEvalHash(b)); + }); + test("changes when a criterion changes", () => { const a = makeEvalFile(); const b = makeEvalFile({ @@ -94,26 +111,110 @@ describe("getEvalHash", () => { describe("computeComparisonFingerprint", () => { test("is deterministic", () => { - const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); - const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh1", + ); assert.equal(fp1, fp2); }); test("changes when the judge model changes", () => { - const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); - const fp2 = computeComparisonFingerprint("claude", "opus", "ph1", "eh1"); + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "opus", + "v1", + "ph1", + "eh1", + ); assert.notEqual(fp1, fp2); }); test("changes when the judge prompt hash changes", () => { - const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); - const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph2", "eh1"); + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph2", + "eh1", + ); + assert.notEqual(fp1, fp2); + }); + + test("changes when the judge CLI version changes — an upgrade is a regime change", () => { + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + "2.1.250", + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "sonnet", + "2.1.251", + "ph1", + "eh1", + ); assert.notEqual(fp1, fp2); }); + test("an unknown judge version is deterministic, not random", () => { + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + undefined, + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "sonnet", + undefined, + "ph1", + "eh1", + ); + assert.equal(fp1, fp2); + }); + test("changes when the eval hash changes", () => { - const fp1 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh1"); - const fp2 = computeComparisonFingerprint("claude", "sonnet", "ph1", "eh2"); + const fp1 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh1", + ); + const fp2 = computeComparisonFingerprint( + "claude", + "sonnet", + "v1", + "ph1", + "eh2", + ); assert.notEqual(fp1, fp2); }); }); @@ -130,6 +231,17 @@ describe("getGitSha / getRepoSlug", () => { }); }); +describe("getJudgePromptHash", () => { + test("hashes the header-stripped prompt — what the judge actually sends", () => { + const raw = readFileSync( + join("src", "eval", "prompts", "criterion-judge.txt"), + "utf8", + ); + assert.equal(getJudgePromptHash(), hashContent(stripPromptHeader(raw))); + assert.notEqual(getJudgePromptHash(), hashContent(raw)); + }); +}); + describe("buildProvenance", () => { test("returns a fully-formed RunProvenance record", () => { const p = buildProvenance(makeEvalFile()); @@ -146,6 +258,7 @@ describe("buildProvenance", () => { const recomputed = computeComparisonFingerprint( p.judgeProvider, p.judgeModel, + p.judgeVersion, p.judgePromptHash, p.evalHash, ); diff --git a/src/tests/eval-trend.test.ts b/src/tests/eval-trend.test.ts index d008926..dc18c53 100644 --- a/src/tests/eval-trend.test.ts +++ b/src/tests/eval-trend.test.ts @@ -38,6 +38,17 @@ describe("parseTrendArgs", () => { test("throws on an invalid --mode value", () => { assert.throws(() => parseTrendArgs(["--mode", "bogus"]), /Invalid --mode/); }); + + test("throws when a flag is missing its value instead of silently ignoring it", () => { + // `--mode` silently falling back to "all" would hand the user + // non-comparable data while they believe it's strict. + assert.throws(() => parseTrendArgs(["--mode"]), /Missing value for --mode/); + assert.throws( + () => parseTrendArgs(["--history", "--mode", "strict"]), + /Missing value for --history/, + ); + assert.throws(() => parseTrendArgs(["--eval"]), /Missing value for --eval/); + }); }); describe("printTrends", () => { diff --git a/src/tests/eval.test.ts b/src/tests/eval.test.ts index 8d42093..08a261c 100644 --- a/src/tests/eval.test.ts +++ b/src/tests/eval.test.ts @@ -464,6 +464,26 @@ describe("judgeOutput", () => { criteria, ); }); + + test("pins the judge model the provenance records, never the CLI default", async () => { + const { judgeOutput } = await import("../eval/judge.js"); + const { resolveJudgeModel } = await import("../eval/provenance.js"); + const { mockDir } = installMockClaude('{"pass": true, "reason": "ok"}'); + + // Extend the mock to record its argv, so we can see the --model flag. + const argsFile = join(mockDir, "args.txt"); + const responseFile = join(mockDir, "response.ndjson"); + writeFileSync( + join(mockDir, "claude"), + `#!/usr/bin/env bash\necho "$@" > "${argsFile}"\ncat "${responseFile}"\nexit 0\n`, + "utf8", + ); + + await judgeOutput("output", "criterion"); + + const argv = readFileSync(argsFile, "utf8"); + assert.match(argv, new RegExp(`--model ${resolveJudgeModel()}`)); + }); }); // --------------------------------------------------------------------------- @@ -861,3 +881,64 @@ test_cases: ); }); }); + +// --------------------------------------------------------------------------- +// single-model identity +// --------------------------------------------------------------------------- + +describe("single-model identity", () => { + let originalArgv: string[]; + let originalPath: string; + let originalModel: string | undefined; + + beforeEach(() => { + originalArgv = process.argv.slice(); + originalPath = process.env["PATH"] ?? ""; + originalModel = process.env["EXECUTANT_MODEL"]; + }); + afterEach(() => { + process.argv.length = 0; + for (const a of originalArgv) process.argv.push(a); + process.env["PATH"] = originalPath; + if (originalModel === undefined) delete process.env["EXECUTANT_MODEL"]; + else process.env["EXECUTANT_MODEL"] = originalModel; + }); + + test("records EXECUTANT_MODEL as the run's model when no --models is given", async () => { + // Regression: the bare path hard-coded "claude/sonnet", filing an + // EXECUTANT_MODEL=haiku run's scores under the sonnet trend series. + const { main } = await import("../eval/index.js"); + + const dir = tmpDir(); + const templatePath = join(dir, "template.txt"); + writeFileSync(templatePath, "Template {{INPUT}}\n", "utf8"); + const evalYaml = ` +name: identity-test +prompt: ${templatePath} +placeholders: + - INPUT +test_cases: + - id: case-one + vars: + INPUT: hello + criteria: + - "Output is non-empty" +`; + const evalFilePath = join(dir, "test.eval.yaml"); + writeFileSync(evalFilePath, evalYaml, "utf8"); + const outputJson = join(dir, "out.json"); + + installMockClaude('{"pass": true, "reason": "ok"}'); + process.env["EXECUTANT_MODEL"] = "haiku"; + + process.argv.length = 0; + for (const a of ["node", "eval", "--output-json", outputJson, evalFilePath]) + process.argv.push(a); + await main(); + + const written = JSON.parse(readFileSync(outputJson, "utf8")); + assert.equal(written.models[0].model, "haiku"); + assert.equal(written.runs[0].model.model, "haiku"); + assert.equal(written.provenance.judgeModel, "haiku"); + }); +}); From 8d701f4fc367b85528ef04f46e8a7a14b0f6ce1b Mon Sep 17 00:00:00 2001 From: Coston Perkins Date: Mon, 31 Aug 2026 12:57:21 -0500 Subject: [PATCH 4/5] feat(eval): single-file HTML report via eval:trend --html MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `npm run eval:trend -- --html ` now also renders the history log as one self-contained HTML file: a leaderboard card per eval (latest run per model, ranked by score then cost, LOCAL/API badges) with expandable per-model run histories carrying cost, duration, git sha, and the same regime-change markers as the terminal view. Themed with the @coston/design-tokens purple-dark theme the TUI already uses (src/ui/theme.ts) — the tokens are CSS-native oklch() strings, so they are injected verbatim as custom properties at generation time. No build step, no script, no external stylesheet, font, or asset: the file works offline and can be attached to a PR or CI artifact as-is. renderHtmlReport is pure (history entries in, HTML string out); the only I/O is the flag handling in trend-index.ts. HistoryEntry is now exported for consumers of loadHistory's return type. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019thBZNhFBYfFP7Dk5cgzMW --- ARCHITECTURE.md | 4 +- docs/eval-comparison.md | 7 + src/eval/history.ts | 2 +- src/eval/html-report.ts | 245 +++++++++++++++++++++++++++++ src/eval/trend-index.ts | 16 +- src/tests/eval-html-report.test.ts | 135 ++++++++++++++++ src/tests/eval-trend.test.ts | 6 + 7 files changed, 412 insertions(+), 3 deletions(-) create mode 100644 src/eval/html-report.ts create mode 100644 src/tests/eval-html-report.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fee7766..a280c6d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -173,7 +173,9 @@ The eval system tests and iteratively refines the prompt templates in `src/promp **`src/eval/history.ts`** — `appendHistory`/`loadHistory`: persist one JSONL record per model/eval per run (score, cost, duration, provenance) to a history log. `buildTrends(entries, mode)`: groups records by eval+model into time-ordered series; `"strict"` keeps only runs matching the group's latest `comparisonFingerprint`, `"all"` keeps every run and flags the points where the fingerprint changed (`regimeChange`). -**`src/eval/trend-index.ts`** — `npm run eval:trend` CLI. Reads a history JSONL file, filters by `--eval`, builds trends in `--mode strict|all`, and renders them via `printTrends`. +**`src/eval/trend-index.ts`** — `npm run eval:trend` CLI. Reads a history JSONL file, filters by `--eval`, builds trends in `--mode strict|all`, renders them via `printTrends`, and with `--html ` also writes the report as a single self-contained HTML file (`html-report.ts`). + +**`src/eval/html-report.ts`** — `renderHtmlReport(entries, mode)`: pure entries-in/HTML-out renderer for `--html`. One leaderboard card per eval (latest run per model, ranked) with expandable per-model run histories and regime-change markers, themed inline with the `@coston/design-tokens` purple-dark theme the TUI uses — no build step, script, or external asset. **`src/eval/report.ts`** — Terminal output: `printRun()` for single-model pass/fail table; `printComparison()` for multi-model side-by-side comparison table; `printTrends()` for the `eval:trend` time-series view, with a marker line at each regime-change point. diff --git a/docs/eval-comparison.md b/docs/eval-comparison.md index 891d715..b0f13b0 100644 --- a/docs/eval-comparison.md +++ b/docs/eval-comparison.md @@ -202,6 +202,7 @@ npm run eval:trend # all history, all runs npm run eval:trend -- --eval judge-evaluation # filter to one eval npm run eval:trend -- --mode strict # only strictly-comparable runs npm run eval:trend -- --history results/other.jsonl # a different history file +npm run eval:trend -- --html results/bench.html # also write a single-file HTML report ``` Two modes: @@ -216,6 +217,12 @@ Two modes: getting better/worse" questions, at the cost of dropping older runs made under a different judge/prompt/eval config. +`--html ` additionally writes the same data as a single self-contained +HTML file — leaderboards per eval (latest run per model) with expandable run +histories and regime markers, themed with the `@coston/design-tokens` +purple-dark theme the TUI uses. No build step and no external assets: the +file works offline and can be attached to a PR or CI artifact as-is. + Each trend point can always be traced back to its `run_at`, `git_sha`, and full judge config via the underlying history record. To keep that guarantee, a run that reuses any cached case result from an existing `--output-csv` diff --git a/src/eval/history.ts b/src/eval/history.ts index 50b4a2e..1978d61 100644 --- a/src/eval/history.ts +++ b/src/eval/history.ts @@ -16,7 +16,7 @@ import { dirname } from "node:path"; import { modelLabel } from "./export.js"; import type { EvalComparison } from "./types.js"; -interface HistoryEntry { +export interface HistoryEntry { runAt: string; repo?: string; gitSha?: string; diff --git a/src/eval/html-report.ts b/src/eval/html-report.ts new file mode 100644 index 0000000..501ae00 --- /dev/null +++ b/src/eval/html-report.ts @@ -0,0 +1,245 @@ +// ============================================================================ +// EVAL HTML REPORT +// ============================================================================ +// Renders the eval history log as a single self-contained HTML file — +// leaderboards per eval (latest run per model) plus the full run history with +// regime-change markers, themed with the same @coston/design-tokens +// purple-dark theme the TUI uses (src/ui/theme.ts). The tokens are stored as +// CSS-native oklch() strings, so they are injected verbatim; no build step, +// no external stylesheet, script, or font — the file works offline as-is. +// +// Everything here is pure: history entries in, HTML string out. The only I/O +// lives at the `--html` flag in trend-index.ts. + +import { createRequire } from "node:module"; +import { formatDuration } from "../lib/utils.js"; +import { buildTrends } from "./history.js"; +import type { HistoryEntry, TrendGroup, TrendMode } from "./history.js"; + +const _require = createRequire(import.meta.url); +const { themes } = _require("@coston/design-tokens/tokens.json") as { + themes: Record>; +}; + +const THEME_NAME = "purple-dark"; // the TUI's theme — keep the two in step + +/** The design-token slots the report consumes, emitted as CSS custom properties. */ +const TOKEN_KEYS = [ + "background", + "foreground", + "card", + "card-foreground", + "primary", + "primary-foreground", + "muted", + "muted-foreground", + "border", + "success", + "warning", + "destructive", + "chart-1", + "chart-2", + "radius", +] as const; + +function themeCss(): string { + const theme = themes[THEME_NAME]!; + return TOKEN_KEYS.map((key) => ` --${key}: ${theme[key]};`).join("\n"); +} + +function esc(s: string): string { + return s + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function fmtCost(costUsd: number | undefined): string { + return costUsd !== undefined ? `$${costUsd.toFixed(4)}` : "—"; +} + +function fmtPct(pct: number): string { + return `${Math.round(pct * 100)}%`; +} + +/** Latest run per model within one eval, ranked by score, then by cost. */ +function leaderboardRows(groups: TrendGroup[]): string { + const latest = groups + .map((g) => g.points.at(-1)!) + .sort((a, b) => b.pct - a.pct || (a.costUsd ?? 0) - (b.costUsd ?? 0)); + + return latest + .map((point, i) => { + const scoreClass = point.pct === 1 ? "score-full" : ""; + return ` + ${i + 1} + ${esc(point.modelLabel)}${point.provider === "opencode" ? "LOCAL" : "API"} + + + ${point.passCount}/${point.totalCriteria} + ${fmtPct(point.pct)} + + ${esc(formatDuration(point.durationMs))} + ${fmtCost(point.costUsd)} + `; + }) + .join("\n"); +} + +/** One run history list per eval+model series, with regime-change markers. */ +function historyRows(group: TrendGroup): string { + return group.points + .map((point) => { + const marker = point.regimeChange + ? `
  • regime change — judge/prompt/eval fingerprint differs from previous run
    • \n` + : ""; + const judge = `judge ${esc(point.judgeProvider)}/${esc(point.judgeModel)}${point.judgeVersion ? ` v${esc(point.judgeVersion)}` : ""} · fingerprint ${esc(point.comparisonFingerprint)}`; + return `${marker}
  • + ${esc(point.runAt.slice(0, 16).replace("T", " "))} + ${esc(point.gitSha?.slice(0, 7) ?? "unknown")} + ${point.passCount}/${point.totalCriteria} · ${fmtPct(point.pct)} + ${fmtCost(point.costUsd)} + ${esc(formatDuration(point.durationMs))} +
    • `; + }) + .join("\n"); +} + +function evalSection(evalName: string, groups: TrendGroup[]): string { + const runCount = groups.reduce((s, g) => s + g.points.length, 0); + const series = groups + .map( + (g) => `
    + ${esc(g.modelLabel)}${g.points.length} run(s) +
      +${historyRows(g)} +
    +
    `, + ) + .join("\n"); + + return `
    +
    +

    ${esc(evalName)}

    +

    ${groups.length} model(s) · ${runCount} recorded run(s). Ranked by the latest run per model; expand a series for its full history.

    +
    +
    + + + +${leaderboardRows(groups)} + +
    #ModelScoreDurationCost
    +
    +${series} +
    `; +} + +/** + * Renders the full report. Groups the history by eval, one leaderboard card + * per eval, each with expandable per-model run histories. + */ +export function renderHtmlReport( + entries: HistoryEntry[], + mode: TrendMode, +): string { + const groups = buildTrends(entries, mode); + const byEval = new Map(); + for (const group of groups) { + const list = byEval.get(group.evalName); + if (list) list.push(group); + else byEval.set(group.evalName, [group]); + } + + const sections = + byEval.size > 0 + ? [...byEval.entries()] + .map(([evalName, evalGroups]) => evalSection(evalName, evalGroups)) + .join("\n") + : `

    No history records yet. Run evals with --history to start tracking.

    `; + + const generatedAt = new Date().toISOString().slice(0, 16).replace("T", " "); + + return ` + + + + +Executant Bench + + + +
    +
    +

    Executant Bench

    +

    Model comparison across executant's prompt-template evals, judged by LLM criteria.

    +
    + generated ${generatedAt} UTC + ${entries.length} history record(s) + mode: ${mode} +
    +
    +${sections} +
    Generated by executant eval:trend --html from the eval history log. A score jump at a regime line is judge/eval drift, not model improvement.
    +
    + + +`; +} diff --git a/src/eval/trend-index.ts b/src/eval/trend-index.ts index 9c74397..43f30ec 100644 --- a/src/eval/trend-index.ts +++ b/src/eval/trend-index.ts @@ -12,8 +12,11 @@ // and prints per eval+model time series with regime-change markers wherever // the judge model/version, judge prompt, or eval spec changed between runs. +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { loadHistory, buildTrends } from "./history.js"; +import { renderHtmlReport } from "./html-report.js"; import { printTrends } from "./report.js"; import type { TrendMode } from "./history.js"; @@ -23,6 +26,7 @@ interface TrendArgs { historyPath: string; mode: TrendMode; evalFilter?: string; + htmlPath?: string; } /** @@ -43,6 +47,7 @@ export function parseTrendArgs(rawArgs: string[]): TrendArgs { let historyPath = DEFAULT_HISTORY_PATH; let mode: TrendMode = "all"; let evalFilter: string | undefined; + let htmlPath: string | undefined; for (let i = 0; i < rawArgs.length; i++) { const arg = rawArgs[i]!; @@ -56,6 +61,7 @@ export function parseTrendArgs(rawArgs: string[]): TrendArgs { " --mode strict|all strict: only runs comparable to the latest judge/prompt/eval config", " all (default): every run, with regime-change markers", " --eval Filter to a single eval_name", + " --html Also write the report as a single self-contained HTML file", ].join("\n"), ); process.exit(0); @@ -71,10 +77,12 @@ export function parseTrendArgs(rawArgs: string[]): TrendArgs { mode = value; } else if (arg === "--eval") { evalFilter = takeValue(rawArgs, ++i, "--eval"); + } else if (arg === "--html") { + htmlPath = takeValue(rawArgs, ++i, "--html"); } } - return { historyPath, mode, evalFilter }; + return { historyPath, mode, evalFilter, htmlPath }; } export async function main(): Promise { @@ -102,6 +110,12 @@ export async function main(): Promise { `\nTrend report (${args.mode === "strict" ? "strict-comparable" : "all runs"} mode) — ${groups.length} eval+model series\n`, ); printTrends(groups); + + if (args.htmlPath) { + mkdirSync(dirname(args.htmlPath), { recursive: true }); + writeFileSync(args.htmlPath, renderHtmlReport(entries, args.mode), "utf8"); + console.log(`Wrote ${args.htmlPath}`); + } } if (process.argv[1] === fileURLToPath(import.meta.url)) { diff --git a/src/tests/eval-html-report.test.ts b/src/tests/eval-html-report.test.ts new file mode 100644 index 0000000..240dbb6 --- /dev/null +++ b/src/tests/eval-html-report.test.ts @@ -0,0 +1,135 @@ +// ============================================================================ +// EVAL HTML REPORT — unit tests +// ============================================================================ +// Tests for src/eval/html-report.ts: renderHtmlReport is pure (entries in, +// HTML string out), so everything is asserted on the returned markup. + +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; + +import { renderHtmlReport } from "../eval/html-report.js"; +import type { HistoryEntry } from "../eval/history.js"; + +function makeEntry(overrides: Partial = {}): HistoryEntry { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "abcdef1234567890abcdef1234567890abcdef12", + evalName: "sample-eval", + modelLabel: "claude/sonnet", + provider: "claude", + model: "sonnet", + passCount: 4, + totalCriteria: 6, + pct: 4 / 6, + costUsd: 0.05, + durationMs: 60_000, + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "prompt-hash-1", + evalHash: "eval-hash-1", + comparisonFingerprint: "fingerprint-1", + ...overrides, + }; +} + +describe("renderHtmlReport", () => { + test("emits a single self-contained document with the purple-dark tokens inline", () => { + const html = renderHtmlReport([makeEntry()], "all"); + assert.match(html, /^/); + assert.match(html, /--background: oklch\(/); + assert.match(html, /--chart-1: oklch\(/); + // Self-contained: no external fetches of any kind. + assert.doesNotMatch(html, /' })], + "all", + ); + assert.doesNotMatch(html, /