diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6a778f2..a280c6d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -159,19 +159,27 @@ Large text passed to Claude lives in `src/prompts/*.txt`. They use `{{PLACEHOLDE The eval system tests and iteratively refines the prompt templates in `src/prompts/`. It is not user-facing — run via `npm run eval` during development. -**`src/eval/index.ts`** — CLI entry point. Parses `--refine`, `--max-iter`, `--models`, `--cases`, `--output-json`, and `--output-csv` flags. Accepts one or more eval file paths as positional arguments. `--cases` accepts comma-separated case IDs or 1-based index ranges (e.g. `simple,1-3`) to run a subset without editing YAML. Single-model mode: loads existing CSV results for resume (skips already-scored cases), runs remaining cases, optional refine loop. Multi-model mode (2+ models via `--models`): runs each model independently, builds an `EvalComparison`, prints a side-by-side table. When multiple files are passed, output paths are auto-suffixed per eval name. +**`src/eval/index.ts`** — CLI entry point. Parses `--refine`, `--max-iter`, `--models`, `--cases`, `--output-json`, `--output-csv`, and `--history` flags. Accepts one or more eval file paths as positional arguments. `--cases` accepts comma-separated case IDs or 1-based index ranges (e.g. `simple,1-3`) to run a subset without editing YAML. Single-model mode: loads existing CSV results for resume (skips already-scored cases), runs remaining cases, optional refine loop. Multi-model mode (2+ models via `--models`): runs each model independently, builds an `EvalComparison`, prints a side-by-side table. When multiple files are passed, output paths are auto-suffixed per eval name. Every `EvalComparison` carries a `provenance` record (see `src/eval/provenance.ts`); `--history ` appends it (plus score/cost/duration) to a JSONL log for `npm run eval:trend`. **`src/eval/load.ts`** — Parses `evals/*.eval.yaml` via Zod. Resolves fixture paths (values in `vars` that end in `.md` / `.txt` are read and substituted with file contents). -**`src/eval/runner.ts`** — `runPrompt(templatePath, vars, model?)`: substitutes `{{PLACEHOLDER}}` vars, runs the prompt through the specified model via `runAgent`, and returns the raw text output. Claude receives `METHODOLOGY` as `appendSystemPrompt`; OpenCode does not (flag not supported). +**`src/eval/runner.ts`** — `runPrompt(templatePath, vars, model?)`: substitutes `{{PLACEHOLDER}}` vars, runs the prompt through the specified model via `runAgent`, and returns `{ output, costUsd? }` — the raw text output plus API cost when the provider reports one (Claude only). Claude receives `METHODOLOGY` as `appendSystemPrompt`; OpenCode does not (flag not supported). **`src/eval/judge.ts`** — `judgeOutput()`: takes a single output string and a criterion string, always uses Claude for judgment (the authoritative judge), and returns `{ pass: boolean, reason: string }`. **`src/eval/refine.ts`** — `refinePrompt()`: given the current template and a list of failures, calls Claude with the prompt-refiner prompt and returns a rewritten template. -**`src/eval/report.ts`** — Terminal output: `printRun()` for single-model pass/fail table; `printComparison()` for multi-model side-by-side comparison table. +**`src/eval/provenance.ts`** — `buildProvenance(evalFile)`: captures `RunProvenance` for a comparison run — `git rev-parse HEAD` / origin remote for repo+SHA, `claude --version` for judge version, and sha256 hashes (truncated to 12 hex chars) of the judge prompt template and the resolved eval spec. `comparisonFingerprint` combines judge provider+model+prompt hash+eval hash into the key that decides whether two runs are strictly comparable. -**`src/eval/export.ts`** — `toJson(comparison)` and `toCsv(comparison)`: serialize `EvalComparison` for benchmark analysis. CSV is denormalized (one row per criterion judgment per model) with columns `eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms`. +**`src/eval/history.ts`** — `appendHistory`/`loadHistory`: persist one JSONL record per model/eval per run (score, cost, duration, provenance) to a history log. `buildTrends(entries, mode)`: groups records by eval+model into time-ordered series; `"strict"` keeps only runs matching the group's latest `comparisonFingerprint`, `"all"` keeps every run and flags the points where the fingerprint changed (`regimeChange`). + +**`src/eval/trend-index.ts`** — `npm run eval:trend` CLI. Reads a history JSONL file, filters by `--eval`, builds trends in `--mode strict|all`, renders them via `printTrends`, and with `--html ` also writes the report as a single self-contained HTML file (`html-report.ts`). + +**`src/eval/html-report.ts`** — `renderHtmlReport(entries, mode)`: pure entries-in/HTML-out renderer for `--html`. One leaderboard card per eval (latest run per model, ranked) with expandable per-model run histories and regime-change markers, themed inline with the `@coston/design-tokens` purple-dark theme the TUI uses — no build step, script, or external asset. + +**`src/eval/report.ts`** — Terminal output: `printRun()` for single-model pass/fail table; `printComparison()` for multi-model side-by-side comparison table; `printTrends()` for the `eval:trend` time-series view, with a marker line at each regime-change point. + +**`src/eval/export.ts`** — `toJson(comparison)` and `toCsv(comparison)`: serialize `EvalComparison` for benchmark analysis. CSV is denormalized (one row per criterion judgment per model) with columns `eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms, cost_usd, run_at, repo, git_sha, judge_provider, judge_model, judge_version, judge_prompt_hash, eval_hash, comparison_fingerprint`. **`src/eval/prompts/`** — Eval-specific prompts (`criterion-judge.txt`, `prompt-refiner.txt`). Same `{{PLACEHOLDER}}` convention as `src/prompts/`. diff --git a/BACKLOG.md b/BACKLOG.md index 32faa13..bf8e101 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -28,6 +28,18 @@ Known improvements deferred from code reviews and audits. - **Vars-aware caching in `resolveWorkflow`** — two `workflow:` steps referencing the same file are currently fetched and parsed independently. A cache would need to key on the resolved path/URL *and* the step's `vars:` override (serialized), since two steps referencing the same file with different overrides must not share a resolved `Workflow` — added complexity that isn't justified without evidence repeated-reference workflows are common. +## Implemented (status bar, 2026-08) + +- ✅ **Status bar wraps to a second line under a long repo name or branch** — `StatusBar` (`src/ui/StatusBar.tsx`) rendered the repo name and branch at their full length with no regard for terminal width; a sufficiently long value (a long feature-branch name, or a checkout directory not named after the repo) wrapped the row onto a second line, breaking the fixed single-line layout the rest of the footer assumes. `fitRepoLabel` (`src/lib/statusline.ts`) now shrinks name and/or branch with an ellipsis to fit the terminal width minus the gauge segment's reserved width, handing a short name's unused budget to the branch. `App.tsx` passes `stdout?.columns ?? 80` down as the new required `columns` prop. Caught via `src/tests/statusline-ui.test.ts`, whose "names the repo and branch" test also hardcoded the literal string "executant" as the expected repo name (true only when the checkout directory happens to be named after the repo) — updated to assert the shape of the segment rather than its exact text, so the suite doesn't assume a specific checkout location. + +## Implemented (subprocess lifecycle, 2026-08) + +- ✅ **`timeout_seconds` on a script step actually kills the command on dash-family shells** — `runCommand` (`src/tasks/command.ts`) spawns `sh -c ""` and, on timeout, called `proc.kill()` against that `sh` process alone. On shells that fork a real child for the command rather than exec-replacing themselves (verified with `dash`, Debian/Ubuntu's default `/bin/sh`, for anything beyond a single tail-callable command — e.g. `sleep 60`), killing only the `sh` PID left the actual command running as an orphan, still holding stdout/stderr open. The step's reader loop was waiting on EOF from that pipe, so it never saw one and hung indefinitely instead of throwing `TimeoutError` — silently defeating the feature on any Linux box where `/bin/sh` is dash (i.e. most). Fixed by spawning `detached: true` (making `proc.pid` the leader of its own process group) and killing the whole group (`process.kill(-proc.pid!, "SIGTERM")`) on timeout — `startTimeout` (`src/tasks/stream.ts`) now takes an optional `onTimeout` override for this, defaulting to the original `proc.kill()` for `claude.ts`/`opencode.ts`, which spawn the CLI binary directly and don't have this problem. Detaching moves the child out of executant's own process group, so `command.ts` also gained explicit `SIGINT`/`SIGTERM`/`SIGHUP` cleanup handlers (mirroring the pattern already used in `claude.ts`/`opencode.ts` for `SIGTERM`/`SIGHUP`) so a terminal Ctrl+C during a running script step still reaches the whole process tree instead of just executant itself. Caught via `src/tests/command.test.ts`'s existing timeout test, which used to hang for the full 60s test-runner timeout on this sandbox rather than failing fast — a new regression test spawns a real detached grandchild and asserts it's gone (not orphaned) after the timeout fires. + +## Implemented (eval comparison observability, 2026-08) + +- ✅ **Run provenance + judge drift markers for eval comparisons** — historical eval-comparison trends were previously opaque: a score change on the same eval/model could come from the model under test, or silently from a judge/prompt/eval regime change, with no way to tell which. Every `EvalComparison` (`src/eval/index.ts`) now carries a `provenance` record (`src/eval/provenance.ts`): `runAt`, `repo`/`gitSha` (from `git`), `judgeProvider`/`judgeModel`/`judgeVersion` (`claude --version`, when readable), and sha256 hashes of the judge prompt template and the resolved eval spec, combined into a `comparisonFingerprint` — the strict-comparability key. Per-case API cost (`TestResult.costUsd`, Claude only — OpenCode/local models don't report one) is captured alongside score and duration and rolled up into `EvalRun.totalCostUsd`. JSON/CSV output (`src/eval/export.ts`) include all of it; CSV repeats the values per row the same way `duration_ms` already did, keeping the pivot-table shape. `--history ` (new flag, wired into `eval:compare`) appends one JSONL record per model/eval per run (`src/eval/history.ts`); `npm run eval:trend` (`src/eval/trend-index.ts`) reads that log and renders per eval+model time series in two modes — `all` (every run, with a marker line wherever the judge/prompt/eval fingerprint changed since the previous run) and `strict` (only runs matching the latest fingerprint). The separate git-worktree-based workflow-eval harness (`src/eval/workflow.ts`) was left untouched — this only covers the criterion-judge comparison system `docs/eval-comparison.md` documents. + ## Implemented (TUI, 2026-08) - ✅ **Context gauge reads per-call usage, not the step's cumulative total** — the gauge shipped reading `output:usage`, which is parsed from the Claude CLI's final `result` message. Those counts are summed across every API call the step made: each turn re-reads the whole cached prefix, so `cache_read_input_tokens` grows by roughly the context size *per turn*. Dividing that by a 200k window is a category error — it compares throughput to capacity. On a long agentic step it read **3781.1k/200k** (~25 turns' worth), with the bar pinned at 100% because `buildGauge` caps `pct`. Verified against a real three-turn `claude -p` transcript: the calls occupied 37,670 → 37,844 → 38,166 tokens while the result message reported 113,680 — exactly the sum of the three. The fix adds a separate `output:context` event emitted from the stream's `assistant` messages (deduplicated by message id, since the CLI emits one per content block sharing that turn's usage) carrying the session's `input + cacheCreation + cacheRead` as of its latest turn; the gauge replaces on each one, and `output:usage` goes back to being purely the run report's throughput total. The gauge is also cleared on a prompt step's `step:start`, so each step's session starts from an empty window rather than inheriting the previous step's fill — one session per step, one gauge per session, never carried across or summed. A side benefit is that the gauge now moves *during* a step rather than only at its end. diff --git a/README.md b/README.md index 979cee4..a167982 100644 --- a/README.md +++ b/README.md @@ -580,7 +580,7 @@ npm run eval -- \ npm run eval -- evals/plan-decompose.eval.yaml evals/judge-evaluation.eval.yaml ``` -The `--output-csv` file is denormalized (one row per criterion judgment per model) — ready for pivot tables and charts. See [docs/eval-comparison.md](docs/eval-comparison.md) for column definitions and interpretation guidance. +The `--output-csv` file is denormalized (one row per criterion judgment per model) — ready for pivot tables and charts. Every run also carries provenance (repo, git SHA, judge model/prompt, eval spec hash) and per-case cost. Pass `--history results/eval-history.jsonl` to accumulate a time series, then `npm run eval:trend` to view it, with markers wherever the judge/prompt/eval config changed. See [docs/eval-comparison.md](docs/eval-comparison.md) for column definitions and interpretation guidance. ### Workflow evals (end-to-end agentic testing) diff --git a/docs/eval-comparison.md b/docs/eval-comparison.md index 754edc9..b0f13b0 100644 --- a/docs/eval-comparison.md +++ b/docs/eval-comparison.md @@ -63,14 +63,47 @@ judge-evaluation — 2 models compared TOTAL 7/9 78% 8/9 89% ``` +## Run provenance and cost + +Every comparison run captures a `provenance` record so historical trends stay +interpretable — a score change can come from the model under test, or from a +change in the judge/eval regime itself, and the two should never be confused: + +| Field | Description | +|---|---| +| `runAt` | ISO timestamp of the run | +| `repo` | `owner/repo`, parsed from the `origin` git remote (GitHub only) | +| `gitSha` | Commit evaluated (`git rev-parse HEAD`) | +| `judgeProvider` / `judgeModel` | The judge is always Claude — this records which model. The judge is pinned to `EXECUTANT_MODEL` (default `sonnet`), never the CLI's configured default, so the recorded model is the one that actually judged | +| `judgeVersion` | `claude --version`, when it can be read | +| `judgePromptHash` | Hash of `src/eval/prompts/criterion-judge.txt` (header-stripped — the text the judge actually receives) | +| `evalHash` | Hash of the resolved eval spec (test cases, vars, criteria) | +| `comparisonFingerprint` | Hash of judge provider+model+version+prompt hash+eval hash — the strict-comparability key | + +Each case's API cost (USD, Claude only — OpenCode/local models don't report +cost) is captured alongside its score and duration on every `TestResult`, and +rolled up into `EvalRun.totalCostUsd`. + ## JSON output format -The `--output-json` file contains the full `EvalComparison` object: +The `--output-json` file contains the full `EvalComparison` object, including +`provenance`: ```json { "evalName": "judge-evaluation", "templatePath": "evals/judge-evaluation.eval.yaml", + "provenance": { + "runAt": "2026-08-31T12:00:00.000Z", + "repo": "coston/executant", + "gitSha": "4c7875ebb732337aba1737e357f2b7ba51f064e2", + "judgeProvider": "claude", + "judgeModel": "sonnet", + "judgeVersion": "2.1.251", + "judgePromptHash": "a1b2c3d4e5f6", + "evalHash": "f6e5d4c3b2a1", + "comparisonFingerprint": "0011223344aa" + }, "models": [ { "provider": "claude", "model": "sonnet" }, { "provider": "opencode", "model": "llama-qwen7b/qwen2.5-coder-7b" } @@ -123,13 +156,20 @@ The `--output-csv` file is **denormalized** — one row per criterion judgment p | `model` | Model name as passed to the CLI | | `pass` | `true` or `false` | | `reason` | Judge's reasoning for the pass/fail verdict | +| `duration_ms` | Wall-clock time for the case | +| `cost_usd` | API cost for the case (Claude only; empty for OpenCode/local models) | +| `run_at`, `repo`, `git_sha`, `judge_provider`, `judge_model`, `judge_version`, `judge_prompt_hash`, `eval_hash`, `comparison_fingerprint` | Run provenance — see [Run provenance and cost](#run-provenance-and-cost) | + +Provenance and cost values repeat across every row of a run, the same way +`duration_ms` already does — the CSV is denormalized for pivot tables, not +optimized for storage. ### Example rows ```csv -eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason -"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","claude/sonnet","claude","sonnet","true","Response is well-formed JSON" -"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","opencode/llama-qwen7b/qwen2.5-coder-7b","opencode","llama-qwen7b/qwen2.5-coder-7b","true","JSON parses without error" +eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms,cost_usd,run_at,repo,git_sha,judge_provider,judge_model,judge_version,judge_prompt_hash,eval_hash,comparison_fingerprint +"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","claude/sonnet","claude","sonnet","true","Response is well-formed JSON",1820,0.0042,"2026-08-31T12:00:00.000Z","coston/executant","4c7875ebb732337aba1737e357f2b7ba51f064e2","claude","sonnet","2.1.251","a1b2c3d4e5f6","f6e5d4c3b2a1","0011223344aa" +"judge-evaluation","evals/judge-evaluation.eval.yaml","clear-pass","Output is valid JSON","opencode/llama-qwen7b/qwen2.5-coder-7b","opencode","llama-qwen7b/qwen2.5-coder-7b","true","JSON parses without error",4310,"","2026-08-31T12:00:00.000Z","coston/executant","4c7875ebb732337aba1737e357f2b7ba51f064e2","claude","sonnet","2.1.251","a1b2c3d4e5f6","f6e5d4c3b2a1","0011223344aa" ``` ### Pivot table recipe (Excel / Google Sheets) @@ -142,6 +182,55 @@ eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason Plot `model_label` on X axis, `pct = pass / total_per_model` on Y axis, grouped by `eval_name`. This gives a quick overview of relative model performance across prompt templates. +## Historical trend reporting + +Pass `--history ` to append one JSONL record per model/eval to a history +log every time you run an eval — score, cost, duration, and the provenance +above (this is what `eval:compare` does automatically, into +`results/eval-history.jsonl`): + +```bash +npm run eval -- --models claude/sonnet,claude/haiku \ + --history results/eval-history.jsonl \ + evals/judge-evaluation.eval.yaml +``` + +Then view the trend for each eval+model series: + +```bash +npm run eval:trend # all history, all runs +npm run eval:trend -- --eval judge-evaluation # filter to one eval +npm run eval:trend -- --mode strict # only strictly-comparable runs +npm run eval:trend -- --history results/other.jsonl # a different history file +npm run eval:trend -- --html results/bench.html # also write a single-file HTML report +``` + +Two modes: + +- **`all`** (default) — every historical run is shown, so no data is ever lost. A + `─── regime change ───` marker line is printed wherever the judge model/version, + judge prompt, or eval spec (`comparison_fingerprint`) changed since the + previous run in that series, so a score jump reads as a regime change rather + than a mysterious improvement or regression. +- **`strict`** — only runs whose `comparison_fingerprint` matches the most + recent run are shown. This is the safe default for "is the model actually + getting better/worse" questions, at the cost of dropping older runs made + under a different judge/prompt/eval config. + +`--html ` additionally writes the same data as a single self-contained +HTML file — leaderboards per eval (latest run per model) with expandable run +histories and regime markers, themed with the `@coston/design-tokens` +purple-dark theme the TUI uses. No build step and no external assets: the +file works offline and can be attached to a PR or CI artifact as-is. + +Each trend point can always be traced back to its `run_at`, `git_sha`, and +full judge config via the underlying history record. To keep that guarantee, +a run that reuses any cached case result from an existing `--output-csv` +skips the history append entirely (the cached scores were produced under the +previous run's provenance) — delete the CSV to re-run and record. A corrupt +line in the history file (e.g. from an interrupted append) is skipped with a +warning rather than aborting the report. + ## Adding a new model Any provider supported by Executant can be added to a comparison run: diff --git a/package.json b/package.json index 98ffc24..7c51916 100644 --- a/package.json +++ b/package.json @@ -22,12 +22,13 @@ "test": "env -u NODE_TEST_CONTEXT -u EXECUTANT_PROVIDER -u EXECUTANT_MODEL -u EXECUTANT_AGENT -u EXECUTANT_REPORT_SUGGESTION -u OTEL_EXPORTER_OTLP_ENDPOINT -u OTEL_EXPORTER_OTLP_TRACES_ENDPOINT -u OTEL_EXPORTER_OTLP_METRICS_ENDPOINT -u TRACEPARENT EXECUTANT_RETROSPECTIVE=0 EXECUTANT_STATUSLINE=0 node --import tsx/esm --test --test-timeout=60000 src/tests/*.test.ts", "eval": "tsx src/eval/index.ts", "eval:workflow": "tsx src/eval/workflow-index.ts", + "eval:trend": "tsx src/eval/trend-index.ts", "setup": "tsx src/setup.ts", "models:download": "tsx src/native-models.ts", "models:start": "tsx src/model-server.ts start", "models:stop": "tsx src/model-server.ts stop", "models:status": "tsx src/model-server.ts status", - "eval:compare": "for f in evals/*.eval.yaml; do npm run eval -- --models claude/opus,claude/sonnet,claude/haiku,opencode/llama-qwen7b/qwen2.5-coder-7b,opencode/llama-qwen14b/qwen2.5-coder-14b,opencode/llama-llama8b/llama-3.1-8b --output-csv \"results/$(basename $f .eval.yaml).csv\" \"$f\"; done && npm run eval:compare:report", + "eval:compare": "for f in evals/*.eval.yaml; do npm run eval -- --models claude/opus,claude/sonnet,claude/haiku,opencode/llama-qwen7b/qwen2.5-coder-7b,opencode/llama-qwen14b/qwen2.5-coder-14b,opencode/llama-llama8b/llama-3.1-8b --output-csv \"results/$(basename $f .eval.yaml).csv\" --history results/eval-history.jsonl \"$f\"; done && npm run eval:compare:report", "eval:compare:report": "tsx src/eval/report-gen.ts", "lint": "eslint src", "knip": "knip" @@ -116,7 +117,8 @@ "src/model-server.ts", "src/eval/index.ts", "src/eval/workflow-index.ts", - "src/eval/report-gen.ts" + "src/eval/report-gen.ts", + "src/eval/trend-index.ts" ], "project": [ "src/**/*.ts", diff --git a/src/eval/export.ts b/src/eval/export.ts index e59dcb6..a82120d 100644 --- a/src/eval/export.ts +++ b/src/eval/export.ts @@ -4,9 +4,15 @@ // Serializes EvalComparison results to JSON and CSV for benchmark analysis. // // CSV columns (one row per criterion judgment): -// eval_name, template_path, case_id, criterion, model_label, provider, model, pass, reason, duration_ms +// eval_name, template_path, case_id, criterion, model_label, provider, model, +// pass, reason, duration_ms, cost_usd, run_at, repo, git_sha, judge_provider, +// judge_model, judge_version, judge_prompt_hash, eval_hash, comparison_fingerprint +// +// Provenance and cost columns repeat the same value across every row of a run +// (like duration_ms already does) — the denormalized shape is optimized for +// pivot tables, not storage efficiency. -import type { EvalComparison, ModelTarget } from "./types.js"; +import type { EvalComparison, ModelTarget, RunProvenance } from "./types.js"; export function modelLabel(m: ModelTarget): string { return m.label ?? `${m.provider}/${m.model}`; @@ -17,6 +23,32 @@ export function toJson(comparison: EvalComparison): string { return JSON.stringify(comparison, null, 2); } +const PROVENANCE_COLUMNS = [ + "run_at", + "repo", + "git_sha", + "judge_provider", + "judge_model", + "judge_version", + "judge_prompt_hash", + "eval_hash", + "comparison_fingerprint", +] as const; + +function provenanceCells(p: RunProvenance): string[] { + return [ + csvCell(p.runAt), + csvCell(p.repo ?? ""), + csvCell(p.gitSha ?? ""), + csvCell(p.judgeProvider), + csvCell(p.judgeModel), + csvCell(p.judgeVersion ?? ""), + csvCell(p.judgePromptHash), + csvCell(p.evalHash), + csvCell(p.comparisonFingerprint), + ]; +} + /** Serializes a comparison to CSV — one row per criterion judgment per model. */ export function toCsv(comparison: EvalComparison): string { const header = [ @@ -30,6 +62,8 @@ export function toCsv(comparison: EvalComparison): string { "pass", "reason", "duration_ms", + "cost_usd", + ...PROVENANCE_COLUMNS, ].join(","); const rows: string[] = [header]; @@ -50,6 +84,8 @@ export function toCsv(comparison: EvalComparison): string { c.pass ? "true" : "false", csvCell(c.reason), String(result.durationMs), + result.costUsd !== undefined ? String(result.costUsd) : "", + ...provenanceCells(comparison.provenance), ].join(","), ); } diff --git a/src/eval/history.ts b/src/eval/history.ts new file mode 100644 index 0000000..1978d61 --- /dev/null +++ b/src/eval/history.ts @@ -0,0 +1,160 @@ +// ============================================================================ +// EVAL HISTORY +// ============================================================================ +// Persists one JSONL record per model/eval per comparison run (score, cost, +// duration, provenance) so trends can be tracked over time, and groups those +// records into per eval+model trend lines for `npm run eval:trend`. +// +// Two trend modes: +// "strict" — only runs whose comparisonFingerprint matches the most recent +// run in the group (judge/prompt/eval unchanged) +// "all" — every historical run, with regime-change points flagged so +// judge/prompt/eval drift is explicit rather than hidden + +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { modelLabel } from "./export.js"; +import type { EvalComparison } from "./types.js"; + +export interface HistoryEntry { + runAt: string; + repo?: string; + gitSha?: string; + evalName: string; + modelLabel: string; + provider: string; + model: string; + passCount: number; + totalCriteria: number; + pct: number; + costUsd?: number; + durationMs: number; + judgeProvider: string; + judgeModel: string; + judgeVersion?: string; + judgePromptHash: string; + evalHash: string; + comparisonFingerprint: string; +} + +/** One row per model run in the comparison, carrying the run's shared provenance. */ +export function toHistoryEntries(comparison: EvalComparison): HistoryEntry[] { + const { provenance } = comparison; + return comparison.runs.map((run) => ({ + runAt: provenance.runAt, + repo: provenance.repo, + gitSha: provenance.gitSha, + evalName: comparison.evalName, + modelLabel: modelLabel(run.model), + provider: run.model.provider, + model: run.model.model, + passCount: run.totalPass, + totalCriteria: run.totalCriteria, + pct: run.totalCriteria === 0 ? 0 : run.totalPass / run.totalCriteria, + costUsd: run.totalCostUsd, + durationMs: run.results.reduce((s, r) => s + r.durationMs, 0), + judgeProvider: provenance.judgeProvider, + judgeModel: provenance.judgeModel, + judgeVersion: provenance.judgeVersion, + judgePromptHash: provenance.judgePromptHash, + evalHash: provenance.evalHash, + comparisonFingerprint: provenance.comparisonFingerprint, + })); +} + +/** Appends one JSONL line per model run in the comparison to `historyPath`. */ +export function appendHistory( + comparison: EvalComparison, + historyPath: string, +): void { + const entries = toHistoryEntries(comparison); + if (entries.length === 0) return; + mkdirSync(dirname(historyPath), { recursive: true }); + const lines = entries.map((e) => JSON.stringify(e)).join("\n") + "\n"; + appendFileSync(historyPath, lines, "utf8"); +} + +/** + * Reads all history records from a JSONL file. Returns [] if the file doesn't + * exist. A line that fails to parse (e.g. a half-written trailing line from an + * interrupted append) is skipped with a warning naming the file and line — + * one corrupt line must never make months of accumulated history unreadable. + */ +export function loadHistory(historyPath: string): HistoryEntry[] { + if (!existsSync(historyPath)) return []; + return readFileSync(historyPath, "utf8") + .split("\n") + .flatMap((line, i) => { + if (!line.trim()) return []; + try { + return [JSON.parse(line) as HistoryEntry]; + } catch (err) { + process.stderr.write( + `[eval:trend] warning: skipping corrupt line ${i + 1} of ${historyPath}: ${err instanceof Error ? err.message : String(err)}\n`, + ); + return []; + } + }); +} + +export type TrendMode = "strict" | "all"; + +export interface TrendPoint extends HistoryEntry { + /** True when this point's comparisonFingerprint differs from the previous point in its group. */ + regimeChange: boolean; +} + +export interface TrendGroup { + evalName: string; + modelLabel: string; + points: TrendPoint[]; +} + +/** + * Groups history entries by eval+model into time-ordered trend lines. + * "strict" keeps only runs matching the group's most recent comparisonFingerprint + * (guaranteed judge/prompt/eval comparability); "all" keeps every run and marks + * the points where the fingerprint changed. + */ +export function buildTrends( + entries: HistoryEntry[], + mode: TrendMode, +): TrendGroup[] { + const byGroup = new Map(); + for (const entry of entries) { + const key = `${entry.evalName}::${entry.modelLabel}`; + const group = byGroup.get(key); + if (group) group.push(entry); + else byGroup.set(key, [entry]); + } + + const groups: TrendGroup[] = []; + for (const groupEntries of byGroup.values()) { + // Identity comes from the entries themselves, not from re-splitting the + // map key — a label that ever contained the delimiter would round-trip + // truncated. + const { evalName, modelLabel: label } = groupEntries[0]!; + const sorted = [...groupEntries].sort((a, b) => + a.runAt.localeCompare(b.runAt), + ); + const latestFingerprint = sorted.at(-1)?.comparisonFingerprint; + const selected = + mode === "strict" + ? sorted.filter((e) => e.comparisonFingerprint === latestFingerprint) + : sorted; + + const points: TrendPoint[] = selected.map((entry, i) => ({ + ...entry, + regimeChange: + i > 0 && + selected[i - 1]!.comparisonFingerprint !== entry.comparisonFingerprint, + })); + groups.push({ evalName, modelLabel: label, points }); + } + + return groups.sort( + (a, b) => + a.evalName.localeCompare(b.evalName) || + a.modelLabel.localeCompare(b.modelLabel), + ); +} diff --git a/src/eval/html-report.ts b/src/eval/html-report.ts new file mode 100644 index 0000000..597439a --- /dev/null +++ b/src/eval/html-report.ts @@ -0,0 +1,283 @@ +// ============================================================================ +// EVAL HTML REPORT +// ============================================================================ +// Renders the eval history log as a single self-contained HTML file — +// leaderboards per eval (latest run per model) plus the full run history with +// regime-change markers, themed with the same @coston/design-tokens +// purple-dark theme the TUI uses (src/ui/theme.ts). The tokens are stored as +// CSS-native oklch() strings, so they are injected verbatim; no build step, +// no external stylesheet, script, or font — the file works offline as-is. +// +// Everything here is pure: history entries in, HTML string out. The only I/O +// lives at the `--html` flag in trend-index.ts. + +import { createRequire } from "node:module"; +import { formatDuration } from "../lib/utils.js"; +import { buildTrends } from "./history.js"; +import type { HistoryEntry, TrendGroup, TrendMode } from "./history.js"; + +const _require = createRequire(import.meta.url); +const { themes } = _require("@coston/design-tokens/tokens.json") as { + themes: Record>; +}; + +const THEME_NAME = "purple-dark"; // the TUI's theme — keep the two in step + +/** The design-token slots the report consumes, emitted as CSS custom properties. */ +const TOKEN_KEYS = [ + "background", + "foreground", + "card", + "card-foreground", + "primary", + "primary-foreground", + "muted", + "muted-foreground", + "border", + "success", + "warning", + "destructive", + "chart-1", + "chart-2", + "radius", +] as const; + +function themeCss(): string { + const theme = themes[THEME_NAME]!; + return TOKEN_KEYS.map((key) => ` --${key}: ${theme[key]};`).join("\n"); +} + +function esc(s: string): string { + return s + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function fmtCost(costUsd: number | undefined): string { + return costUsd !== undefined ? `$${costUsd.toFixed(4)}` : "—"; +} + +function fmtPct(pct: number): string { + return `${Math.round(pct * 100)}%`; +} + +/** Latest run per model within one eval, ranked by score, then by cost. */ +function leaderboardRows(groups: TrendGroup[]): string { + const latest = groups + .map((g) => g.points.at(-1)!) + .sort((a, b) => b.pct - a.pct || (a.costUsd ?? 0) - (b.costUsd ?? 0)); + + return latest + .map((point, i) => { + const scoreClass = point.pct === 1 ? "score-full" : ""; + return ` + ${i + 1} + ${esc(point.modelLabel)}${point.provider === "opencode" ? "LOCAL" : "API"} + + + ${point.passCount}/${point.totalCriteria} + ${fmtPct(point.pct)} + + ${esc(formatDuration(point.durationMs))} + ${fmtCost(point.costUsd)} + `; + }) + .join("\n"); +} + +/** One run history list per eval+model series, with regime-change markers. */ +function historyRows(group: TrendGroup): string { + return group.points + .map((point) => { + const marker = point.regimeChange + ? `
  • regime change — judge/prompt/eval fingerprint differs from previous run
    • \n` + : ""; + const judge = `judge ${esc(point.judgeProvider)}/${esc(point.judgeModel)}${point.judgeVersion ? ` v${esc(point.judgeVersion)}` : ""} · fingerprint ${esc(point.comparisonFingerprint)}`; + return `${marker}
  • + ${esc(point.runAt.slice(0, 16).replace("T", " "))} + ${esc(point.gitSha?.slice(0, 7) ?? "unknown")} + ${point.passCount}/${point.totalCriteria} · ${fmtPct(point.pct)} + ${fmtCost(point.costUsd)} + ${esc(formatDuration(point.durationMs))} +
    • `; + }) + .join("\n"); +} + +/** + * A 10-bin score-distribution strip over each model's latest run — shows at a + * glance whether the eval separates models or everyone clusters at a ceiling. + * Pure CSS bars (no script) to keep the file self-contained; per-bin counts + * live in title tooltips and every score is also in the table, so the strip + * is a summary, never the sole carrier. + */ +function scoreHistogram(groups: TrendGroup[]): string { + const latest = groups.map((g) => g.points.at(-1)!.pct); + const bins = Array.from( + { length: 10 }, + (_, i) => + latest.filter((p) => p >= i / 10 && (i === 9 ? p <= 1 : p < (i + 1) / 10)) + .length, + ); + const max = Math.max(...bins, 1); + const bars = bins + .map((count, i) => { + const h = count === 0 ? 0 : Math.max(18, (count / max) * 100); + const label = `${i * 10}–${i * 10 + 10}%: ${count} model(s)`; + return ``; + }) + .join(""); + return ` `; +} + +function evalSection(evalName: string, groups: TrendGroup[]): string { + const runCount = groups.reduce((s, g) => s + g.points.length, 0); + const series = groups + .map( + (g) => `
    + ${esc(g.modelLabel)}${g.points.length} run(s) +
      +${historyRows(g)} +
    +
    `, + ) + .join("\n"); + + return `
    +
    +
    +

    ${esc(evalName)}

    +

    ${groups.length} model(s) · ${runCount} recorded run(s). Ranked by the latest run per model; expand a series for its full history.

    +
    +${scoreHistogram(groups)} +
    +
    + + + +${leaderboardRows(groups)} + +
    #ModelScoreDurationCost
    +
    +${series} +
    `; +} + +/** + * Renders the full report. Groups the history by eval, one leaderboard card + * per eval, each with expandable per-model run histories. + */ +export function renderHtmlReport( + entries: HistoryEntry[], + mode: TrendMode, +): string { + const groups = buildTrends(entries, mode); + const byEval = new Map(); + for (const group of groups) { + const list = byEval.get(group.evalName); + if (list) list.push(group); + else byEval.set(group.evalName, [group]); + } + + const sections = + byEval.size > 0 + ? [...byEval.entries()] + .map(([evalName, evalGroups]) => evalSection(evalName, evalGroups)) + .join("\n") + : `

    No history records yet. Run evals with --history to start tracking.

    `; + + const generatedAt = new Date().toISOString().slice(0, 16).replace("T", " "); + + return ` + + + + +Executant Bench + + + +
    +
    +

    Executant Bench

    +

    Model comparison across executant's prompt-template evals, judged by LLM criteria.

    +
    + generated ${generatedAt} UTC + ${entries.length} history record(s) + mode: ${mode} +
    +
    +${sections} +
    Generated by executant eval:trend --html from the eval history log. A score jump at a regime line is judge/eval drift, not model improvement.
    +
    + + +`; +} diff --git a/src/eval/index.ts b/src/eval/index.ts index eeb3332..13cfed6 100644 --- a/src/eval/index.ts +++ b/src/eval/index.ts @@ -29,6 +29,9 @@ import { printDiff, } from "./report.js"; import { toJson, toCsv, modelLabel } from "./export.js"; +import { DEFAULT_MODEL } from "../lib/utils.js"; +import { buildProvenance } from "./provenance.js"; +import { appendHistory } from "./history.js"; import type { EvalArgs, EvalRun, @@ -37,6 +40,7 @@ import type { FailureContext, ModelTarget, ModelEvalRun, + RunProvenance, TestResult, } from "./types.js"; @@ -101,6 +105,9 @@ export function loadExistingResults( const pass = cells[col["pass"]] === "true"; const reason = cells[col["reason"]] ?? ""; const durationMs = parseInt(cells[col["duration_ms"]] ?? "0", 10); + const costCell = cells[col["cost_usd"]]; + const costUsd = + costCell !== undefined && costCell !== "" ? Number(costCell) : undefined; if (!byModel.has(label)) byModel.set(label, new Map()); const byCase = byModel.get(label)!; @@ -113,6 +120,7 @@ export function loadExistingResults( passCount: 0, failCount: 0, durationMs, + costUsd, }); } const result = byCase.get(caseId)!; @@ -198,6 +206,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { let outputJson: string | undefined; let outputCsv: string | undefined; let caseFilter: string | undefined; + let historyPath: string | undefined; for (let i = 0; i < rawArgs.length; i++) { const arg = rawArgs[i]!; @@ -215,6 +224,8 @@ export function parseArgs(rawArgs: string[]): EvalArgs { outputCsv = rawArgs[++i]; } else if (arg === "--cases" && rawArgs[i + 1]) { caseFilter = rawArgs[++i]; + } else if (arg === "--history" && rawArgs[i + 1]) { + historyPath = rawArgs[++i]; } else if (!arg.startsWith("-")) { evalFiles.push(arg); } @@ -232,6 +243,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { " --cases Run a subset of cases: IDs or index ranges, e.g. simple,1-3", " --output-json Write comparison JSON to file", " --output-csv Write comparison CSV to file (supports resume)", + " --history Append a JSONL history record for trend tracking (see `npm run eval:trend`)", ].join("\n"), ); process.exit(0); @@ -251,6 +263,7 @@ export function parseArgs(rawArgs: string[]): EvalArgs { models, outputJson, outputCsv, + historyPath, }; } @@ -270,19 +283,22 @@ async function runEval( ? applyCaseFilter(evalFile.testCases, caseFilter) : evalFile.testCases; const results: TestResult[] = []; + let cachedCount = 0; for (const tc of cases) { const hit = cached?.get(tc.id); if (hit) { process.stdout.write(` skipping ${tc.id} (cached)\n`); results.push(hit); + cachedCount++; continue; } process.stdout.write(` running ${tc.id}…`); const start = performance.now(); let output: string; + let costUsd: number | undefined; try { - output = await runPrompt(path, tc.vars, model); + ({ output, costUsd } = await runPrompt(path, tc.vars, model)); } catch (err) { const durationMs = Math.round(performance.now() - start); const msg = `run error: ${err instanceof Error ? err.message : String(err)}`; @@ -313,12 +329,18 @@ async function runEval( passCount, failCount, durationMs, + costUsd, }); process.stdout.write(` ${passCount}/${criteria.length}\n`); } const totalPass = results.reduce((s, r) => s + r.passCount, 0); const totalCriteria = results.reduce((s, r) => s + r.criteria.length, 0); + const costs = results + .map((r) => r.costUsd) + .filter((c): c is number => typeof c === "number"); + const totalCostUsd = + costs.length > 0 ? costs.reduce((a, b) => a + b, 0) : undefined; return { evalName: evalFile.name, @@ -326,9 +348,34 @@ async function runEval( results, totalPass, totalCriteria, + totalCostUsd, + cachedCount, }; } +/** + * Appends to the history log only when every result in the comparison was + * freshly executed. A resumed run reuses scores produced under the previous + * run's provenance (different time, commit, possibly a different judge), so + * appending them under today's provenance would fabricate a trend point. + */ +export function recordHistory( + comparison: EvalComparison, + historyPath: string, +): void { + const cachedTotal = comparison.runs.reduce( + (sum, run) => sum + (run.cachedCount ?? 0), + 0, + ); + if (cachedTotal > 0) { + console.log( + ` Skipping history append: ${cachedTotal} case result(s) were reused from the output CSV, so this run's provenance does not describe them. Delete the CSV to re-run and record.`, + ); + return; + } + appendHistory(comparison, historyPath); +} + export function collectFailures( run: EvalRun, evalFile: ReturnType, @@ -380,6 +427,7 @@ function buildComparisonTable( async function runMultiModelEval( evalFile: ReturnType, models: ModelTarget[], + provenance: RunProvenance, existingCsv?: string, caseFilter?: string, ): Promise { @@ -405,6 +453,7 @@ async function runMultiModelEval( models, runs, comparisonTable: buildComparisonTable(runs), + provenance, }; } @@ -463,6 +512,8 @@ async function runEvalFile( ? deriveOutputPath(args.outputJson, evalFile.name) : args.outputJson; + const provenance = buildProvenance(evalFile); + // Multi-model comparison mode if (args.models.length > 1) { if (args.refine) { @@ -473,6 +524,7 @@ async function runEvalFile( const comparison = await runMultiModelEval( evalFile, args.models, + provenance, outputCsv, args.caseFilter, ); @@ -480,37 +532,42 @@ async function runEvalFile( if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); + if (args.historyPath) recordHistory(comparison, args.historyPath); return; } - // Single-model mode — load cached results for resume support - const singleModel = args.models[0]; + // Single-model mode — load cached results for resume support. When no + // --models was given, resolve the same default runPrompt will actually use + // (EXECUTANT_MODEL, then sonnet) and pass it explicitly, so the recorded + // model label always names the model that ran — previously + // `EXECUTANT_MODEL=haiku` runs were filed under "claude/sonnet". + const model = args.models[0] ?? { + provider: "claude" as const, + model: process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL, + }; const existing = outputCsv ? loadExistingResults(outputCsv) : new Map(); - const label = singleModel ? modelLabel(singleModel) : "claude/sonnet"; let run = await runEval( evalFile, undefined, - singleModel, - existing.get(label), + model, + existing.get(modelLabel(model)), args.caseFilter, ); printRun(run); // Write output files (wraps single-model run in a minimal comparison) - if (outputJson || outputCsv) { - const model = singleModel ?? { - provider: "claude" as const, - model: "sonnet", - }; + if (outputJson || outputCsv || args.historyPath) { const comparison: EvalComparison = { evalName: evalFile.name, templatePath: evalFile.prompt, models: [model], runs: [{ ...run, model }], comparisonTable: buildComparisonTable([{ ...run, model }]), + provenance, }; if (outputJson) writeOutputFile(outputJson, toJson(comparison)); if (outputCsv) writeOutputFile(outputCsv, toCsv(comparison)); + if (args.historyPath) recordHistory(comparison, args.historyPath); } if (!args.refine || run.totalPass === run.totalCriteria) return; @@ -528,13 +585,7 @@ async function runEvalFile( saveRefinedTemplate(evalFile.prompt, improved); printRefinementHeader(iter, args.maxIter); - run = await runEval( - evalFile, - undefined, - singleModel, - undefined, - args.caseFilter, - ); + run = await runEval(evalFile, undefined, model, undefined, args.caseFilter); printRun(run); if (run.totalPass > bestRun.totalPass) { diff --git a/src/eval/judge.ts b/src/eval/judge.ts index cd49300..7ae33f5 100644 --- a/src/eval/judge.ts +++ b/src/eval/judge.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import { z } from "zod"; import { runClaudeStructured } from "../tasks/claude.js"; import { stripPromptHeader } from "../lib/utils.js"; +import { resolveJudgeModel } from "./provenance.js"; import type { CriterionResult } from "./types.js"; const __dir = dirname(fileURLToPath(import.meta.url)); @@ -40,6 +41,11 @@ export async function judgeOutput( type: "claude", name: "eval:criterion-judge", prompt, + // Pinned so the judge that runs is the judge provenance records — + // without this the CLI falls back to the user's configured default + // model, and two machines with different defaults would share a + // "strictly comparable" fingerprint for genuinely different judges. + model: resolveJudgeModel(), allowedTools: [], permissionMode: "default", }, diff --git a/src/eval/provenance.ts b/src/eval/provenance.ts new file mode 100644 index 0000000..7ae02ef --- /dev/null +++ b/src/eval/provenance.ts @@ -0,0 +1,147 @@ +// ============================================================================ +// EVAL PROVENANCE +// ============================================================================ +// Captures the "what produced this score" context for a comparison run: +// repo + commit evaluated, judge identity, and hashes of the judge prompt and +// eval spec. `comparisonFingerprint` is the stable key that tells historical +// trend reporting whether two runs are strictly comparable. + +import { execSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { DEFAULT_MODEL, stripPromptHeader } from "../lib/utils.js"; +import type { EvalFile, RunProvenance } from "./types.js"; + +/** + * The model the criterion judge runs with. Shared with judge.ts, which pins + * this model on its Claude calls — provenance must record the judge that + * actually ran, not an assumption about it. + */ +export function resolveJudgeModel(): string { + return process.env["EXECUTANT_MODEL"] ?? DEFAULT_MODEL; +} + +const __dir = dirname(fileURLToPath(import.meta.url)); + +function tryExec(cmd: string): string | undefined { + try { + const out = execSync(cmd, { stdio: ["ignore", "pipe", "ignore"] }) + .toString() + .trim(); + return out || undefined; + } catch { + return undefined; + } +} + +/** Commit SHA currently checked out, or undefined outside a git repo. */ +export function getGitSha(): string | undefined { + return tryExec("git rev-parse HEAD"); +} + +/** "owner/repo" parsed from the origin remote URL (GitHub only), or undefined. */ +export function getRepoSlug(): string | undefined { + const url = tryExec("git config --get remote.origin.url"); + if (!url) return undefined; + const match = /github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?$/.exec(url); + return match?.[1]; +} + +const UNSET = Symbol("unset"); +let cachedJudgeVersion: string | undefined | typeof UNSET = UNSET; + +/** + * The judge CLI's version string, when it can be read (e.g. "2.1.251"). + * Memoized per process — every eval-comparison run only pays for one + * `claude --version` spawn no matter how many eval files it covers. + */ +function getJudgeVersion(): string | undefined { + if (cachedJudgeVersion !== UNSET) return cachedJudgeVersion; + const out = tryExec("claude --version"); + cachedJudgeVersion = out?.split(/\s+/)[0]; + return cachedJudgeVersion; +} + +/** Stable short hash (sha256, truncated) of arbitrary content. */ +export function hashContent(content: string): string { + return createHash("sha256").update(content).digest("hex").slice(0, 12); +} + +/** + * Hash of the judge prompt template — changes here signal a judging-regime + * change. Hashes the header-stripped text (what judge.ts actually sends), so + * editing only the documentation header does not flag a false regime change. + */ +export function getJudgePromptHash(): string { + const promptPath = join(__dir, "prompts", "criterion-judge.txt"); + return hashContent(stripPromptHeader(readFileSync(promptPath, "utf8"))); +} + +/** Sorts an object's keys so semantically-equal vars hash identically regardless of YAML key order. */ +function sortedVars(vars: Record): Record { + return Object.fromEntries( + Object.entries(vars).sort(([a], [b]) => a.localeCompare(b)), + ); +} + +/** Hash of the resolved eval spec (test cases + criteria) — changes here signal an eval-regime change. */ +export function getEvalHash(evalFile: EvalFile): string { + const canonical = JSON.stringify({ + name: evalFile.name, + placeholders: [...evalFile.placeholders].sort(), + testCases: [...evalFile.testCases] + .sort((a, b) => a.id.localeCompare(b.id)) + .map((tc) => ({ + id: tc.id, + vars: sortedVars(tc.vars), + criteria: tc.criteria, + })), + }); + return hashContent(canonical); +} + +/** + * Stable fingerprint of judge+prompt+eval config — the strict-comparability + * key. Includes the judge CLI version: a CLI upgrade can change judging + * behavior (new default snapshot, new system prompt), so runs across an + * upgrade are a regime change, not a comparable series. + */ +export function computeComparisonFingerprint( + judgeProvider: string, + judgeModel: string, + judgeVersion: string | undefined, + judgePromptHash: string, + evalHash: string, +): string { + return hashContent( + `${judgeProvider}:${judgeModel}:${judgeVersion ?? "unknown"}:${judgePromptHash}:${evalHash}`, + ); +} + +/** Builds the full provenance record for a comparison run over the given eval file. */ +export function buildProvenance(evalFile: EvalFile): RunProvenance { + const judgeProvider = "claude"; + const judgeModel = resolveJudgeModel(); + const judgeVersion = getJudgeVersion(); + const judgePromptHash = getJudgePromptHash(); + const evalHash = getEvalHash(evalFile); + return { + runAt: new Date().toISOString(), + repo: getRepoSlug(), + gitSha: getGitSha(), + judgeProvider, + judgeModel, + judgeVersion, + judgePromptHash, + evalHash, + comparisonFingerprint: computeComparisonFingerprint( + judgeProvider, + judgeModel, + judgeVersion, + judgePromptHash, + evalHash, + ), + }; +} diff --git a/src/eval/report.ts b/src/eval/report.ts index 5e900d4..35889bf 100644 --- a/src/eval/report.ts +++ b/src/eval/report.ts @@ -1,6 +1,8 @@ import type { EvalComparison, EvalRun, TestResult } from "./types.js"; +import type { TrendGroup } from "./history.js"; import { modelLabel } from "./export.js"; import { theme } from "../ui/theme.js"; +import { formatDuration } from "../lib/utils.js"; const USE_COLOR = Boolean(process.stdout.isTTY) && !process.env["NO_COLOR"]; @@ -159,3 +161,40 @@ export function printComparison(comparison: EvalComparison): void { }); console.log(` ${"TOTAL".padEnd(caseColWidth)} ${totalCells.join("")}\n`); } + +/** + * Prints time-series trend lines for `npm run eval:trend` — one section per + * eval+model group, oldest run first, with a marker row wherever the judge + * model/version, judge prompt, or eval spec changed since the previous run + * (so score jumps caused by a regime change read as a regime change, not a + * mysterious improvement or regression). + */ +export function printTrends(groups: TrendGroup[]): void { + for (const group of groups) { + console.log( + `\n${accent(group.evalName)} — ${dim(group.modelLabel)} (${group.points.length} run(s))\n`, + ); + for (const point of group.points) { + if (point.regimeChange) { + console.log( + warning( + " ─── regime change: judge/prompt/eval fingerprint differs from previous run ───", + ), + ); + } + const pctInt = Math.round(point.pct * 100); + const colorFn = + point.pct === 1 ? pass : point.pct >= 0.5 ? warning : fail; + const score = colorFn( + `${point.passCount}/${point.totalCriteria} ${pctInt}%`, + ); + const cost = + point.costUsd !== undefined ? `$${point.costUsd.toFixed(4)}` : "n/a"; + const sha = point.gitSha ? point.gitSha.slice(0, 7) : "unknown"; + console.log( + ` ${dim(point.runAt)} ${accent(sha)} ${score} ${dim(cost)} ${dim(formatDuration(point.durationMs))}`, + ); + } + } + console.log(); +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts index ce31249..5421d2e 100644 --- a/src/eval/runner.ts +++ b/src/eval/runner.ts @@ -18,16 +18,23 @@ export function substituteVars( ); } +/** Result of running a prompt template through a model. */ +interface PromptRunResult { + output: string; + /** API cost in USD, when the provider reports it (Claude only — OpenCode/local models don't). */ + costUsd?: number; +} + /** * Runs a prompt template with substituted vars through the specified model (no tools). * Defaults to Claude/sonnet when no model target is provided. - * Returns the full text output as a string. + * Returns the full text output plus cost, when the provider reports one. */ export async function runPrompt( templatePath: string, vars: Record, model?: ModelTarget, -): Promise { +): Promise { const template = stripPromptHeader(readFileSync(templatePath, "utf8")); const prompt = substituteVars(template, vars); @@ -35,6 +42,7 @@ export async function runPrompt( const isOpenCode = provider === "opencode"; const lines: string[] = []; + let costUsd: number | undefined; for await (const event of runAgent({ type: "claude", name: `eval:${basename(templatePath, ".txt")}`, @@ -52,7 +60,8 @@ export async function runPrompt( ...(!isOpenCode ? { appendSystemPrompt: METHODOLOGY } : {}), })) { if (event.type === "output:text") lines.push(event.text); + else if (event.type === "output:cost") costUsd = event.usd; } - return lines.join(""); + return { output: lines.join(""), costUsd }; } diff --git a/src/eval/trend-index.ts b/src/eval/trend-index.ts new file mode 100644 index 0000000..43f30ec --- /dev/null +++ b/src/eval/trend-index.ts @@ -0,0 +1,129 @@ +#!/usr/bin/env node +// ============================================================================ +// EVAL:TREND — Historical trend reporting over eval-history.jsonl +// ============================================================================ +// Usage: +// npm run eval:trend +// npm run eval:trend -- --history results/eval-history.jsonl +// npm run eval:trend -- --mode strict +// npm run eval:trend -- --eval judge-evaluation +// +// Reads the JSONL history log produced by `npm run eval -- --history ` +// and prints per eval+model time series with regime-change markers wherever +// the judge model/version, judge prompt, or eval spec changed between runs. + +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadHistory, buildTrends } from "./history.js"; +import { renderHtmlReport } from "./html-report.js"; +import { printTrends } from "./report.js"; +import type { TrendMode } from "./history.js"; + +const DEFAULT_HISTORY_PATH = "results/eval-history.jsonl"; + +interface TrendArgs { + historyPath: string; + mode: TrendMode; + evalFilter?: string; + htmlPath?: string; +} + +/** + * Returns the value at `i`, or throws when the flag has none (end of args, or + * the next token is another flag). A silently-ignored `--mode strict` would + * hand the user non-comparable data while they believe it's strict — the one + * silent failure this tool exists to prevent. + */ +function takeValue(rawArgs: string[], i: number, flag: string): string { + const value = rawArgs[i]; + if (value === undefined || value.startsWith("--")) { + throw new Error(`Missing value for ${flag}`); + } + return value; +} + +export function parseTrendArgs(rawArgs: string[]): TrendArgs { + let historyPath = DEFAULT_HISTORY_PATH; + let mode: TrendMode = "all"; + let evalFilter: string | undefined; + let htmlPath: string | undefined; + + for (let i = 0; i < rawArgs.length; i++) { + const arg = rawArgs[i]!; + if (arg === "--help" || arg === "-h") { + console.log( + [ + "Usage: npm run eval:trend -- [OPTIONS]", + "", + "Options:", + ` --history JSONL history file to read (default: ${DEFAULT_HISTORY_PATH})`, + " --mode strict|all strict: only runs comparable to the latest judge/prompt/eval config", + " all (default): every run, with regime-change markers", + " --eval Filter to a single eval_name", + " --html Also write the report as a single self-contained HTML file", + ].join("\n"), + ); + process.exit(0); + } else if (arg === "--history") { + historyPath = takeValue(rawArgs, ++i, "--history"); + } else if (arg === "--mode") { + const value = takeValue(rawArgs, ++i, "--mode"); + if (value !== "strict" && value !== "all") { + throw new Error( + `Invalid --mode "${value}": expected "strict" or "all"`, + ); + } + mode = value; + } else if (arg === "--eval") { + evalFilter = takeValue(rawArgs, ++i, "--eval"); + } else if (arg === "--html") { + htmlPath = takeValue(rawArgs, ++i, "--html"); + } + } + + return { historyPath, mode, evalFilter, htmlPath }; +} + +export async function main(): Promise { + const args = parseTrendArgs(process.argv.slice(2)); + const allEntries = loadHistory(args.historyPath); + const entries = allEntries.filter( + (e) => !args.evalFilter || e.evalName === args.evalFilter, + ); + + if (allEntries.length === 0) { + console.log( + `No history found at ${args.historyPath}. Run evals with "--history ${args.historyPath}" to start tracking trends.`, + ); + return; + } + if (entries.length === 0) { + console.log( + `No records match --eval "${args.evalFilter}" in ${args.historyPath} (${allEntries.length} record(s) for: ${[...new Set(allEntries.map((e) => e.evalName))].join(", ")}).`, + ); + return; + } + + const groups = buildTrends(entries, args.mode); + console.log( + `\nTrend report (${args.mode === "strict" ? "strict-comparable" : "all runs"} mode) — ${groups.length} eval+model series\n`, + ); + printTrends(groups); + + if (args.htmlPath) { + mkdirSync(dirname(args.htmlPath), { recursive: true }); + writeFileSync(args.htmlPath, renderHtmlReport(entries, args.mode), "utf8"); + console.log(`Wrote ${args.htmlPath}`); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch((err) => { + console.error( + "eval:trend error:", + err instanceof Error ? err.message : String(err), + ); + process.exit(1); + }); +} diff --git a/src/eval/types.ts b/src/eval/types.ts index c288a3b..d58fd56 100644 --- a/src/eval/types.ts +++ b/src/eval/types.ts @@ -24,6 +24,8 @@ export interface TestResult { passCount: number; failCount: number; durationMs: number; + /** API cost in USD for generating this case's output. Undefined when the provider doesn't report cost (e.g. OpenCode/local models). */ + costUsd?: number; } export interface EvalRun { @@ -32,6 +34,15 @@ export interface EvalRun { results: TestResult[]; totalPass: number; totalCriteria: number; + /** Sum of results[].costUsd. Undefined when no result reported a cost. */ + totalCostUsd?: number; + /** + * How many results came from a resumed --output-csv rather than running. + * Non-zero blocks history appends: a cached score was produced under the + * *old* run's provenance, so stamping it with a fresh one would fabricate + * a trend point. + */ + cachedCount?: number; } export interface FailureContext { @@ -60,6 +71,25 @@ export interface ComparisonRow { scores: Record; } +/** + * Provenance metadata for a single eval-comparison run, captured so historical + * trends stay interpretable: which repo/commit was evaluated, which judge + * (model + prompt) scored it, and which eval spec was run. `comparisonFingerprint` + * is a stable hash of judge+prompt+eval config — two runs only belong on the + * same "strict comparable" trend line when it matches. + */ +export interface RunProvenance { + runAt: string; // ISO timestamp + repo?: string; // "owner/repo", when a GitHub remote is configured + gitSha?: string; // commit evaluated + judgeProvider: string; + judgeModel: string; + judgeVersion?: string; // judge CLI version, when it can be determined + judgePromptHash: string; + evalHash: string; // hash of the resolved eval spec (test cases + criteria) + comparisonFingerprint: string; // hash of judgeProvider+judgeModel+judgePromptHash+evalHash +} + /** Full multi-model comparison result for a single eval file. */ export interface EvalComparison { evalName: string; @@ -67,6 +97,7 @@ export interface EvalComparison { models: ModelTarget[]; runs: ModelEvalRun[]; comparisonTable: ComparisonRow[]; + provenance: RunProvenance; } export interface EvalArgs { @@ -82,6 +113,8 @@ export interface EvalArgs { outputJson?: string; /** File path to write comparison CSV to (optional). */ outputCsv?: string; + /** File path to append a JSONL history record to, for trend tracking (optional). */ + historyPath?: string; } // --------------------------------------------------------------------------- diff --git a/src/lib/statusline.ts b/src/lib/statusline.ts index 2077f35..8f0357e 100644 --- a/src/lib/statusline.ts +++ b/src/lib/statusline.ts @@ -125,6 +125,39 @@ export interface RepoInfo { branch?: string; } +/** Truncates to fit, appending an ellipsis rather than silently cutting off. */ +function truncateText(s: string, maxWidth: number): string { + if (maxWidth <= 0) return ""; + if (s.length <= maxWidth) return s; + if (maxWidth === 1) return "…"; + return s.slice(0, maxWidth - 1) + "…"; +} + +/** Width of the 3-space gap StatusBar renders between the repo name and branch. */ +const REPO_BRANCH_GAP = 3; + +/** + * Shrinks repo name and branch to fit within `maxWidth` combined, so a long + * branch name (or, in an unusually-named checkout, a long directory name) + * never wraps the status bar onto a second line. A short name keeps its full + * length and any width it doesn't use is handed to the branch. + */ +export function fitRepoLabel(repo: RepoInfo, maxWidth: number): RepoInfo { + if (!repo.branch) return { name: truncateText(repo.name, maxWidth) }; + + const available = maxWidth - REPO_BRANCH_GAP; + if (repo.name.length + repo.branch.length <= available) return repo; + if (available <= 0) return { name: truncateText(repo.name, maxWidth) }; + + const half = Math.floor(available / 2); + const nameWidth = Math.min(repo.name.length, half); + const branchWidth = Math.max(1, available - nameWidth); + return { + name: truncateText(repo.name, nameWidth), + branch: truncateText(repo.branch, branchWidth), + }; +} + /** Runs a git subcommand in `cwd`, resolving undefined on any failure. */ function runGit( args: string[], diff --git a/src/tasks/command.ts b/src/tasks/command.ts index de60593..0902278 100644 --- a/src/tasks/command.ts +++ b/src/tasks/command.ts @@ -32,9 +32,32 @@ export async function* runCommand(task: CommandTask): AsyncGenerator { const proc = spawn("sh", ["-c", task.command], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, ...traceparentEnv() }, + // `sh -c ""` forks a real child for the command on shells like + // dash whenever it isn't the single tail-callable command in the script + // — killing just the sh PID then leaves that child running and holding + // stdout/stderr open, so a reader waiting on EOF never sees one (this is + // why a timed-out step used to hang instead of stopping). Detached makes + // proc.pid the leader of its own process group, so signalling -proc.pid + // reaches sh and every process it forked. + detached: true, }); - const timeout = startTimeout(proc, task.name, task.timeoutSeconds); + // Detaching moves the child out of executant's own process group, so it no + // longer receives a terminal's Ctrl+C (SIGINT) for free the way a + // non-detached child would — these mirror that by killing the group + // explicitly whenever executant itself is being torn down. + const killGroup = (): void => { + try { + process.kill(-proc.pid!, "SIGTERM"); + } catch { + /* already dead, or never got a pid */ + } + }; + process.once("SIGINT", killGroup); + process.once("SIGTERM", killGroup); + process.once("SIGHUP", killGroup); + + const timeout = startTimeout(proc, task.name, task.timeoutSeconds, killGroup); try { for await (const line of mergeStreamsToLines(proc.stdout!, proc.stderr!)) { @@ -53,5 +76,8 @@ export async function* runCommand(task: CommandTask): AsyncGenerator { } } finally { timeout.cancel(); + process.off("SIGINT", killGroup); + process.off("SIGTERM", killGroup); + process.off("SIGHUP", killGroup); } } diff --git a/src/tasks/stream.ts b/src/tasks/stream.ts index 0446312..4872d78 100644 --- a/src/tasks/stream.ts +++ b/src/tasks/stream.ts @@ -103,16 +103,23 @@ export function waitForExit(proc: ReturnType): Promise { * Arms a kill-on-timeout guard for a child process. * Call check() after waitForExit() to throw TimeoutError if the timer fired. * Call cancel() in a finally block to disarm the timer on normal completion. + * `onTimeout`, when given, replaces the default `proc.kill()` — e.g. to kill + * a whole process group instead of just the immediate child (see command.ts). */ export function startTimeout( proc: ReturnType, taskName: string, timeoutSeconds: number | undefined, + onTimeout?: () => void, ): { check: () => void; cancel: () => void } { if (timeoutSeconds == null) return { check: () => {}, cancel: () => {} }; let timedOut = false; const timer = setTimeout(() => { timedOut = true; + if (onTimeout) { + onTimeout(); + return; + } try { proc.kill(); } catch { diff --git a/src/tests/app-non-tty-ui.test.ts b/src/tests/app-non-tty-ui.test.ts index 9412c84..2cb7108 100644 --- a/src/tests/app-non-tty-ui.test.ts +++ b/src/tests/app-non-tty-ui.test.ts @@ -23,6 +23,7 @@ // followed by workflow:complete) gets batched by React into too few commits // to ever land on the vulnerable "step running on a non-TTY" render at all. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe } from "node:test"; import assert from "node:assert/strict"; import { EventEmitter } from "node:events"; diff --git a/src/tests/app-output-pane-ui.test.ts b/src/tests/app-output-pane-ui.test.ts index aba563d..24da751 100644 --- a/src/tests/app-output-pane-ui.test.ts +++ b/src/tests/app-output-pane-ui.test.ts @@ -11,6 +11,7 @@ // channel that hook listens on, but asserting exact on-screen row positions // from a text frame is what the pure math tests already cover directly. +import "./force-non-ci.js"; // must evaluate before any ink import — see its header import { test, describe } from "node:test"; import assert from "node:assert/strict"; import React from "react"; diff --git a/src/tests/command.test.ts b/src/tests/command.test.ts index ef46eda..e0b4e11 100644 --- a/src/tests/command.test.ts +++ b/src/tests/command.test.ts @@ -5,6 +5,9 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; +import { readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runCommand } from "../tasks/command.js"; import type { @@ -44,6 +47,20 @@ async function collectEventsExpectingError( } } +/** Polls until signal-0 delivery fails with ESRCH (process fully reaped). */ +async function processGone(pid: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + process.kill(pid, 0); + } catch { + return true; + } + await new Promise((r) => setTimeout(r, 50)); + } + return false; +} + // ---------------------------------------------------------------------------- // runCommand // ---------------------------------------------------------------------------- @@ -133,6 +150,41 @@ describe("runCommand — timeout_seconds", () => { assert.equal((error as TimeoutError).exitCode, 3); }); + test("kills the whole process group on timeout, not just the sh wrapper", async () => { + // Regression: `sh -c ""` forks a real child for the command + // itself on shells like dash whenever it isn't the single tail-callable + // command in the script. Killing only the `sh` PID used to leave that + // child (here, `sleep`) running and holding stdout open, so the reader + // never saw EOF and the step hung until the outer test-runner timeout + // rather than throwing TimeoutError. + const pidFile = join( + tmpdir(), + `executant-cmd-timeout-test-${process.pid}-${Date.now()}.pid`, + ); + const task: CommandTask = { + type: "command", + name: "nested-sleep", + command: `sleep 60 & echo $! > ${pidFile}; wait`, + timeoutSeconds: 0.1, + }; + try { + const { error } = await collectEventsExpectingError(task); + assert.ok(error instanceof TimeoutError); + + const grandchildPid = Number(readFileSync(pidFile, "utf8").trim()); + // The group SIGTERM has been sent by the time TimeoutError surfaces, + // but the grandchild can linger briefly as an unreaped zombie — and + // signal 0 still succeeds on a zombie — so poll for it to disappear + // rather than asserting on the very first probe. + assert.ok( + await processGone(grandchildPid, 2000), + "the sleep grandchild should have been killed, not left orphaned", + ); + } finally { + rmSync(pidFile, { force: true }); + } + }); + test("does not throw TimeoutError when command completes before timeout", async () => { const task: CommandTask = { type: "command", diff --git a/src/tests/eval-comparison.test.ts b/src/tests/eval-comparison.test.ts index 15441b4..26c7025 100644 --- a/src/tests/eval-comparison.test.ts +++ b/src/tests/eval-comparison.test.ts @@ -20,6 +20,7 @@ import type { EvalComparison, ModelEvalRun, ModelTarget, + RunProvenance, } from "../eval/types.js"; // ---------------------------------------------------------------------------- @@ -121,6 +122,20 @@ describe("parseArgs — models / output flags", () => { assert.equal(args.outputCsv, undefined); }); + test("--history is parsed", () => { + const args = parseArgs([ + "--history", + "results/eval-history.jsonl", + "evals/test.yaml", + ]); + assert.equal(args.historyPath, "results/eval-history.jsonl"); + }); + + test("historyPath is undefined by default", () => { + const args = parseArgs(["evals/test.yaml"]); + assert.equal(args.historyPath, undefined); + }); + test("all new flags coexist with existing flags", () => { const args = parseArgs([ "--refine", @@ -175,6 +190,20 @@ describe("modelLabel", () => { // Fixture helpers // ---------------------------------------------------------------------------- +function makeProvenance(): RunProvenance { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "abc123def456abc123def456abc123def456abc", + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "hash-prompt-1", + evalHash: "hash-eval-1", + comparisonFingerprint: "hash-fingerprint-1", + }; +} + function makeComparison(): EvalComparison { const claudeModel: ModelTarget = { provider: "claude", model: "sonnet" }; const ocModel: ModelTarget = { @@ -277,6 +306,7 @@ function makeComparison(): EvalComparison { }, }, ], + provenance: makeProvenance(), }; } @@ -322,7 +352,7 @@ describe("toCsv", () => { const lines = csv.trim().split("\n"); assert.equal( lines[0], - "eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms", + "eval_name,template_path,case_id,criterion,model_label,provider,model,pass,reason,duration_ms,cost_usd,run_at,repo,git_sha,judge_provider,judge_model,judge_version,judge_prompt_hash,eval_hash,comparison_fingerprint", ); }); @@ -355,6 +385,31 @@ describe("toCsv", () => { const csv = toCsv(c); assert.ok(csv.includes('"failed, ""badly"""')); }); + + test("rows carry the comparison's provenance columns", () => { + const c = makeComparison(); + const csv = toCsv(c); + assert.ok(csv.includes('"coston/executant"')); + assert.ok(csv.includes('"abc123def456abc123def456abc123def456abc"')); + assert.ok(csv.includes('"hash-fingerprint-1"')); + }); + + test("cost_usd is empty when a result has no cost", () => { + const c = makeComparison(); + const csv = toCsv(c); + const lines = csv.trim().split("\n"); + // header: ...,duration_ms,cost_usd,run_at,... + const costIdx = lines[0]!.split(",").indexOf("cost_usd"); + const firstDataRow = lines[1]!.split(","); + assert.equal(firstDataRow[costIdx], ""); + }); + + test("cost_usd is serialized when a result reports a cost", () => { + const c = makeComparison(); + c.runs[0]!.results[0]!.costUsd = 0.0123; + const csv = toCsv(c); + assert.ok(csv.includes(",0.0123,")); + }); }); // ---------------------------------------------------------------------------- @@ -369,6 +424,7 @@ describe("loadExistingResults", () => { test("round-trips toCsv output back into TestResult objects", async () => { const c = makeComparison(); + c.runs[0]!.results[0]!.costUsd = 0.05; const csv = toCsv(c); // Write to a temp file @@ -392,6 +448,7 @@ describe("loadExistingResults", () => { assert.equal(caseA.passCount, 1); assert.equal(caseA.failCount, 1); assert.equal(caseA.durationMs, 1200); + assert.equal(caseA.costUsd, 0.05); // Check opencode model case-b const ocResults = byModel.get("opencode/llama-qwen7b/qwen2.5-coder-7b"); diff --git a/src/tests/eval-history.test.ts b/src/tests/eval-history.test.ts new file mode 100644 index 0000000..5912d00 --- /dev/null +++ b/src/tests/eval-history.test.ts @@ -0,0 +1,280 @@ +// ============================================================================ +// EVAL HISTORY — unit tests +// ============================================================================ +// Tests for src/eval/history.ts: appendHistory/loadHistory persistence and +// buildTrends grouping, strict-vs-all filtering, and regime-change markers. + +import { test, describe, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { appendFileSync, mkdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + toHistoryEntries, + appendHistory, + loadHistory, + buildTrends, +} from "../eval/history.js"; +import { recordHistory } from "../eval/index.js"; +import type { EvalComparison, RunProvenance } from "../eval/types.js"; + +const _cleanupDirs: string[] = []; +afterEach(() => { + for (const d of _cleanupDirs.splice(0)) + rmSync(d, { recursive: true, force: true }); +}); + +function tmpDir(): string { + const dir = join( + tmpdir(), + `eval-history-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + ); + mkdirSync(dir, { recursive: true }); + _cleanupDirs.push(dir); + return dir; +} + +function makeProvenance(overrides: Partial = {}): RunProvenance { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "sha1", + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "prompt-hash-1", + evalHash: "eval-hash-1", + comparisonFingerprint: "fingerprint-1", + ...overrides, + }; +} + +function makeComparison( + overrides: Partial = {}, +): EvalComparison { + const model = { provider: "claude" as const, model: "sonnet" }; + return { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + models: [model], + runs: [ + { + evalName: "sample-eval", + templatePath: "/fake/prompt.txt", + model, + results: [ + { + caseId: "case-1", + output: "out", + criteria: [{ criterion: "C1", pass: true, reason: "ok" }], + passCount: 1, + failCount: 0, + durationMs: 500, + costUsd: 0.01, + }, + ], + totalPass: 1, + totalCriteria: 1, + totalCostUsd: 0.01, + }, + ], + comparisonTable: [], + provenance: makeProvenance(), + ...overrides, + }; +} + +describe("toHistoryEntries", () => { + test("produces one entry per model run, carrying provenance", () => { + const c = makeComparison(); + const entries = toHistoryEntries(c); + assert.equal(entries.length, 1); + assert.equal(entries[0]!.evalName, "sample-eval"); + assert.equal(entries[0]!.modelLabel, "claude/sonnet"); + assert.equal(entries[0]!.passCount, 1); + assert.equal(entries[0]!.totalCriteria, 1); + assert.equal(entries[0]!.pct, 1); + assert.equal(entries[0]!.costUsd, 0.01); + assert.equal(entries[0]!.durationMs, 500); + assert.equal(entries[0]!.gitSha, "sha1"); + assert.equal(entries[0]!.comparisonFingerprint, "fingerprint-1"); + }); +}); + +describe("appendHistory / loadHistory", () => { + test("round-trips entries through a JSONL file", () => { + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory(makeComparison(), historyPath); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 1); + assert.equal(loaded[0]!.evalName, "sample-eval"); + }); + + test("appends across multiple calls rather than overwriting", () => { + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory( + makeComparison({ + provenance: makeProvenance({ runAt: "2026-01-01T00:00:00.000Z" }), + }), + historyPath, + ); + appendHistory( + makeComparison({ + provenance: makeProvenance({ runAt: "2026-01-02T00:00:00.000Z" }), + }), + historyPath, + ); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 2); + }); + + test("loadHistory returns [] when the file doesn't exist", () => { + assert.deepEqual(loadHistory("/nonexistent/history.jsonl"), []); + }); + + test("creates parent directories as needed", () => { + const dir = tmpDir(); + const historyPath = join(dir, "nested", "dir", "history.jsonl"); + appendHistory(makeComparison(), historyPath); + assert.equal(loadHistory(historyPath).length, 1); + }); + + test("recordHistory refuses to append when any result was resumed from a CSV", () => { + // A cached score was produced under the *previous* run's provenance; + // stamping it with today's runAt/gitSha/fingerprint fabricates a trend + // point that never happened. + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + + const cached = makeComparison(); + cached.runs = [{ ...cached.runs[0]!, cachedCount: 1 }]; + recordHistory(cached, historyPath); + assert.deepEqual(loadHistory(historyPath), []); + + recordHistory(makeComparison(), historyPath); + assert.equal(loadHistory(historyPath).length, 1); + }); + + test("skips a corrupt line instead of losing the whole log", () => { + // A half-written trailing line is what an interrupted append leaves + // behind; it must cost one warning, not the entire history. + const dir = tmpDir(); + const historyPath = join(dir, "history.jsonl"); + appendHistory(makeComparison(), historyPath); + appendFileSync(historyPath, '{"runAt":"2026-01-02T00:0', "utf8"); + + const loaded = loadHistory(historyPath); + assert.equal(loaded.length, 1); + assert.equal(loaded[0]!.evalName, "sample-eval"); + }); +}); + +describe("buildTrends", () => { + test("groups entries by eval+model and sorts by runAt", () => { + const entries = [ + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-03T00:00:00.000Z", + }, + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-01T00:00:00.000Z", + }, + { + ...toHistoryEntries(makeComparison())[0]!, + runAt: "2026-01-02T00:00:00.000Z", + }, + ]; + const groups = buildTrends(entries, "all"); + assert.equal(groups.length, 1); + assert.deepEqual( + groups[0]!.points.map((p) => p.runAt), + [ + "2026-01-01T00:00:00.000Z", + "2026-01-02T00:00:00.000Z", + "2026-01-03T00:00:00.000Z", + ], + ); + }); + + test("separates different evals and models into distinct groups", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + base, + { ...base, evalName: "other-eval" }, + { ...base, modelLabel: "claude/opus" }, + ]; + const groups = buildTrends(entries, "all"); + assert.equal(groups.length, 3); + }); + + test("all mode keeps every run and flags regime-change points", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { + ...base, + runAt: "2026-01-01T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-02T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-03T00:00:00.000Z", + comparisonFingerprint: "fp-b", + }, + ]; + const [group] = buildTrends(entries, "all"); + assert.equal(group!.points.length, 3); + assert.deepEqual( + group!.points.map((p) => p.regimeChange), + [false, false, true], + ); + }); + + test("strict mode keeps only runs matching the latest fingerprint", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { + ...base, + runAt: "2026-01-01T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-02T00:00:00.000Z", + comparisonFingerprint: "fp-a", + }, + { + ...base, + runAt: "2026-01-03T00:00:00.000Z", + comparisonFingerprint: "fp-b", + }, + ]; + const [group] = buildTrends(entries, "strict"); + assert.equal(group!.points.length, 1); + assert.equal(group!.points[0]!.runAt, "2026-01-03T00:00:00.000Z"); + assert.equal(group!.points[0]!.regimeChange, false); + }); + + test("returns groups sorted by evalName then modelLabel", () => { + const base = toHistoryEntries(makeComparison())[0]!; + const entries = [ + { ...base, evalName: "zeta-eval" }, + { ...base, evalName: "alpha-eval" }, + ]; + const groups = buildTrends(entries, "all"); + assert.deepEqual( + groups.map((g) => g.evalName), + ["alpha-eval", "zeta-eval"], + ); + }); +}); diff --git a/src/tests/eval-html-report.test.ts b/src/tests/eval-html-report.test.ts new file mode 100644 index 0000000..915751b --- /dev/null +++ b/src/tests/eval-html-report.test.ts @@ -0,0 +1,171 @@ +// ============================================================================ +// EVAL HTML REPORT — unit tests +// ============================================================================ +// Tests for src/eval/html-report.ts: renderHtmlReport is pure (entries in, +// HTML string out), so everything is asserted on the returned markup. + +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; + +import { renderHtmlReport } from "../eval/html-report.js"; +import type { HistoryEntry } from "../eval/history.js"; + +function makeEntry(overrides: Partial = {}): HistoryEntry { + return { + runAt: "2026-01-01T00:00:00.000Z", + repo: "coston/executant", + gitSha: "abcdef1234567890abcdef1234567890abcdef12", + evalName: "sample-eval", + modelLabel: "claude/sonnet", + provider: "claude", + model: "sonnet", + passCount: 4, + totalCriteria: 6, + pct: 4 / 6, + costUsd: 0.05, + durationMs: 60_000, + judgeProvider: "claude", + judgeModel: "sonnet", + judgeVersion: "2.1.251", + judgePromptHash: "prompt-hash-1", + evalHash: "eval-hash-1", + comparisonFingerprint: "fingerprint-1", + ...overrides, + }; +} + +describe("renderHtmlReport", () => { + test("emits a single self-contained document with the purple-dark tokens inline", () => { + const html = renderHtmlReport([makeEntry()], "all"); + assert.match(html, /^/); + assert.match(html, /--background: oklch\(/); + assert.match(html, /--chart-1: oklch\(/); + // Self-contained: no external fetches of any kind. + assert.doesNotMatch(html, /' })], + "all", + ); + assert.doesNotMatch(html, /