From 445e5a440d1a7c9db5c699e6f415dcc0402d591a Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Wed, 10 Jun 2026 16:18:53 -0400 Subject: [PATCH 1/5] docs(methodology): add harness review findings + precision/judge-scope/contamination disclosures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds docs/methodology_review_findings.md — the full pre-launch methodology review of the evaluation harness (14 findings across critical/high/medium with fix status and file references) — and extends docs/methodology.md with three disclosure sections it calls for: - Statistical precision: 26 tasks, per-track N as low as 5; per-track gaps within plausible noise - Judge scope: the LLM judge scores against gold material, not raw fixture CSVs; numbers are verified by the deterministic C1 scorer - Benchmark contamination: fixtures regenerate per numbered benchmark version; scores comparable only within a grade_version Also documents the cross-family judge-selection policy in the C2 scoring bullet (Opus 4.8 judges all non-Claude candidates; GPT-5.5 xhigh judges Claude candidates) with its two residual caveats. --- docs/methodology.md | 31 +++- docs/methodology_review_findings.md | 223 ++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 docs/methodology_review_findings.md diff --git a/docs/methodology.md b/docs/methodology.md index d679bbb..9a6fc9d 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -186,7 +186,15 @@ The scoring pipeline: - **C1 (fact scoring)** — compares structured numeric outputs against `gold_facts` with absolute or relative tolerance. - **C2 (rubric scoring)** — a judge model evaluates each of the six dimensions using - task-authored per-dimension guidance. + task-authored per-dimension guidance. Judge selection is **cross-family**: Claude + Opus 4.8 judges every candidate except Claude-family models, which are judged by + GPT-5.5 at xhigh reasoning effort instead — no judge ever shares a model family + (and hence a house style) with the model it grades. Two caveats are inherent to + this design and disclosed here: (1) both judges are themselves benchmark + contestants, though neither ever judges its own family; (2) Claude rows are scored + by a different judge than all other rows, so a systematic harshness difference + between the two judges would shift the judged dimensions (40% of the composite) + for those rows relative to the rest of the leaderboard. - **C3 (claim validation)** — deterministic string-matching (with optional judge fallback) checks `gold_insights` presence, `forbidden_claims` absence, and `required_limitations` presence. @@ -227,5 +235,26 @@ response consistency for education program analytics tasks using realistic synth are validated in `ground_truth.json`; a different seed produces different realized values that may shift task difficulty. +**Statistical precision.** GRADE V1 contains 26 tasks (11 operations, 5 outcomes, +10 equity & research), with per-track task counts as low as 5. Overall composites are +averaged over 26 tasks × 5 runs; per-track and per-pack scores rest on much smaller +samples and should be read as indicative rather than precise. Small leaderboard gaps +(roughly a few points) on per-track views are within plausible noise. + +**Judge scope.** The LLM judge evaluates the judged dimensions against the task's gold +material (gold facts, gold insights, required limitations, forbidden claims, reference +outline) — it does **not** see the raw fixture CSVs. Verifying numbers against the data +is the deterministic C1 scorer's job; the judge assesses interpretation quality relative +to the gold material. This keeps judge prompts bounded and judging reproducible, at the +cost that `evidence_linkage` is assessed on plausibility against gold material rather +than re-derived from the data. + +**Benchmark contamination.** Fixture data, gold facts, and task prompts are public in +this repository, so future model training runs may ingest them. Because every dataset is +produced by the synthetic generator (`benchmark/datagen/`), GRADE mitigates this by +regenerating fixtures (new seed, recomputed `ground_truth.json`) for each numbered +benchmark version; scores are only comparable within a benchmark version, and the +version is recorded in every result (`grade_version`). + The public/private boundary governing what can be published is documented in [`docs/publication_policy.md`](publication_policy.md). diff --git a/docs/methodology_review_findings.md b/docs/methodology_review_findings.md new file mode 100644 index 0000000..c69f951 --- /dev/null +++ b/docs/methodology_review_findings.md @@ -0,0 +1,223 @@ +# GRADE Methodology Review — Findings & Fix Plan + +**Date:** 2026-06-10 +**Scope:** Full review of the evaluation harness (`runner/`, `benchmark/rubrics/`, `scripts/run_models.py`) ahead of the first real leaderboard run (issue PearlEng/platform-ai#724). +**Status legend:** `FIXED` (this session) · `OPEN` (assigned to follow-up session) · `DISCLOSE` (methodology-page disclosure, not a code change) + +--- + +## Critical (launch-blocking) + +### C-1. Model registry mislabels which models actually run — `FIXED` +`SEED_MODELS` in `runner/adapters/openrouter_adapter.py` mapped shorthands to +stale placeholder slugs: `"gpt-5"` → `openai/gpt-4o`, `"gemini-2.5-pro"` → +`google/gemini-pro-1.5`, `"claude-opus-4-7"` → `anthropic/claude-opus-4.5`. +A leaderboard published from these would attribute GPT-4o scores to GPT-5. +The adapter's default model and the judge default also used dash-form slugs +(`anthropic/claude-sonnet-4-5`) that do not exist on OpenRouter (the live +slugs use dots, e.g. `anthropic/claude-sonnet-4.6`). + +**Fix applied:** `SEED_MODELS` replaced with the verified launch lineup (see +"Launch model lineup" below; every slug checked against the live +`/api/v1/models` endpoint on 2026-06-10). Adapter default model and judge +default corrected to live slugs. +**Follow-up (separate repo):** `grade-app/arena/pairs.py` +`SEED_MODEL_DISPLAY_NAMES` must be updated to match the new lineup when real +results are imported (part of #724). + +### C-2. Forgetting `--judge` silently produces a fake leaderboard — `FIXED` +Without a judge client, `insight_quality`, `evidence_linkage`, and +`structure_usability` (40% of the composite) were scored as a flat `0.5` for +every model by `_NullJudge`, and nothing in the result recorded this. A +misconfigured run would yield a plausible-looking leaderboard ranked only by +fact-matching. + +**Fix applied:** +- `runner/dispatcher.py` now stamps a `null_judge` flag into every task's + `scorer_flags` when no live judge is configured. +- `runner/cli.py` and `scripts/run_models.py` now **refuse to run** the + `openrouter` adapter without `--judge` unless `--allow-null-judge` is + passed explicitly (stub adapter unaffected; smoke tests keep working). + +### C-3. Judge reliability: invalid slug, silent zeros, no retry — `FIXED` (partially `DISCLOSE`) +`benchmark/rubrics/judge_client.py`: +- `DEFAULT_JUDGE_MODEL` was `anthropic/claude-opus-4-5` — an invalid slug + (live slug is `anthropic/claude-opus-4.5`), so every live judge call would + have failed. +- A judge reply that didn't parse as a float silently became `0.0` — a judge + hiccup charged against the candidate model, with no log, retry, or flag. + +**Fix applied:** +- Default judge slug corrected to a live dot-form slug (now + `anthropic/claude-opus-4.8` per the selection policy below). +- Response parsing now extracts the first float via regex (tolerates stray + punctuation/markdown), retries the API call once on parse failure, and + raises `JudgeScoreError` if both attempts fail — so the failure surfaces in + `failures.json` instead of silently zeroing a dimension. +- `--judge-model` flag added to both CLIs so the judge is configurable per + run. + +**Judge selection policy (decided 2026-06-10, revised same day, implemented):** +the strongest available model judges everyone — the principle being that a +weaker model should not grade a stronger one — and **no judge ever shares a +model family with its candidate**, so house style cannot bias the judged +dimensions. Default judge is **Claude Opus 4.8**; every **Claude-family +candidate** (Opus, Sonnet, Haiku) is instead judged by **GPT-5.5 at xhigh +reasoning effort**. Implemented in +`benchmark.rubrics.judge_client.select_judge_model()` and applied +automatically by both CLIs (`--judge-model` overrides). Reasoning-effort +passthrough was added to `post_chat_completion` for this (the GPT-5.5 judge +sends `reasoning: {effort: "xhigh"}` with an 8192-token budget, since +reasoning tokens share the budget with the one-float answer). + +**`DISCLOSE` — two residual caveats for the methodology page:** +1. *Contestant judges*: both judges are themselves contestants (Opus 4.8 is + judged by GPT-5.5; GPT-5.5 rows are judged by Opus 4.8). Neither ever + judges its own family, but neither is a disinterested third party either. +2. *Calibration mismatch*: Claude rows are scored by a different judge than + all other rows. If the two judges differ in harshness, the Claude rows + sit on a slightly different scale for the three judged dimensions (40% of + composite). Mitigation if it proves material: run both judges over a + small overlap sample and report (or correct for) the judge-to-judge + offset. + +--- + +## High (affects score validity — fix before the real run) + +### H-1. Grounding accuracy is credit-only and key-agnostic → rewards number spam — `OPEN` +`benchmark/rubrics/fact_scoring.py`: a gold fact is credited if **any** +number anywhere in the output (up to 50 prose sentences, plus a `/100` +percent-normalized variant of every candidate) falls within tolerance. +Leniency floors widen the window further (gold count 30 with authored +tolerance 0 gets ±0.6, so an unrelated "29.6%" anywhere credits it). There is +no penalty for wrong/hallucinated numbers. Net effect: verbose, number-dense +outputs systematically outscore concise ones on the highest-weighted +dimension (0.35–0.40). + +**Suggested fix:** require the matching number to appear in a sentence with +token overlap against the gold claim (or within N tokens of a claim keyword), +and/or add a precision-style penalty for confidently asserted numbers that +match no gold fact. At minimum, log per-fact `method` labels into the result +for auditability. + +### H-2. Non-numeric gold facts effectively require verbatim echo — `OPEN` +`fact_scoring.py:score_fact` finds a candidate finding by substring +containment but then scores it with **exact** string equality +(`score_exact_match`), so a finding that contains the claim plus any other +words scores 0. The module docstring promises token-overlap matching that is +not implemented. Non-numeric facts are near-universal misses for all models. + +**Suggested fix:** score the substring/token-overlap match directly (like +C3's `TOKEN_OVERLAP_THRESHOLD` approach) instead of exact equality. + +### H-3. `runs=1` gives every model a free 10% (consistency) — `OPEN` +All three consistency sub-metrics default to 1.0 with a single run. The +leaderboard run must use `--runs ≥ 3` (default 5 is good). Suggested fix: +stamp a `consistency_trivial` scorer flag when `runs < 2`, or exclude the +dimension from the composite in that case. + +### H-4. `max_tokens=1024` truncates analyses; reasoning models break — `OPEN` +`OpenRouterAdapter.DEFAULT_MAX_TOKENS = 1024` is tight for a multi-section +prose analysis; limitations sections come last and get cut first (deflating +`calibration_limitation_handling`). `finish_reason` is not checked, so +truncation is invisible. Reasoning models (GPT-5.5 at high effort) can burn +the entire budget on reasoning tokens and return an empty visible response. + +**Suggested fix:** raise the default to ≥ 4096, check +`choices[0].finish_reason == "length"` and stamp a `truncated` flag, and add +a `reasoning` parameter passthrough (see H-5). + +### H-5. No reasoning-effort support — required for the GPT-5.5 sweep — `OPEN` (partial) +The launch plan includes GPT-5.5 at xhigh/high/medium/low effort. +**Done (2026-06-10):** `post_chat_completion` now accepts a +`reasoning_effort` parameter (used by the GPT-5.5 judge). +**Still open:** `OpenRouterAdapter` (the candidate-side path) does not yet +accept/forward a reasoning effort, and results need distinct `model_id` +labels per effort level (e.g. `openai/gpt-5.5@xhigh`) so leaderboard rows +don't collide. Wire the adapter through the same parameter and add a +`--reasoning-effort` CLI flag (or per-model syntax in `--models`). + +--- + +## Medium (quality / cost — assigned to follow-up session) + +### M-1. C1 never sees the `limitations` field — `FIXED` +`runner/dispatcher.py:_score_c1_grounding` now passes the output's +`limitations` list to `score_facts` as the documented fallback search target. +Covered by `test_c1_grounding_searches_limitations`. + +### M-2. Half the judge spend is wasted — `FIXED` +`score_rubric` now accepts a `dimensions` subset (full-rubric weight +validation unchanged), and the dispatcher judges only the three C2-owned +dimensions (`insight_quality`, `evidence_linkage`, `structure_usability`). +This halves judge calls: 3 per run instead of 6. Covered by +`test_dimensions_subset_judges_only_those` and +`test_c2_judge_not_called_for_non_c2_dimensions`. + +### M-3. Small task count → noisy rankings — `DISCLOSED` +26 tasks total (11 operations, 5 outcomes, 10 equity_research); per-track N +is as low as 5. Disclosure added to `docs/methodology.md` ("Statistical +precision") and to the grade-app public methodology page (treat close calls +as ties). Bootstrap CIs over per-task composites remain a nice-to-have. + +### M-4. Public fixtures start the contamination clock — `DISCLOSED` +Disclosure + policy added to `docs/methodology.md` ("Benchmark +contamination"): fixtures are regenerated (new seed + recomputed +`ground_truth.json`) per numbered benchmark version, and scores are only +comparable within a version. Plain-language version added to the grade-app +methodology page. + +### M-5. Judge cannot verify evidence against data — `DISCLOSED` +Disclosure added to `docs/methodology.md` ("Judge scope") and, in plain +language, to the grade-app methodology page: the judge scores against gold +material; numbers are verified separately by the deterministic C1 scorer. + +### M-6. Bare-float judging (no rationale) — `FIXED` +The judge prompt now asks for a brief (2-3 sentence) justification followed +by a final `SCORE: ` line; parsing prefers the last `SCORE:` line and +falls back to the last float, so numbers quoted in the justification are +never mistaken for the verdict. Non-reasoning judge `max_tokens` raised +16 → 384 to fit the rationale. Locked in before the first official run, so +no published scores change. + +--- + +## Launch model lineup (verified against OpenRouter 2026-06-10) + +| Leaderboard entry | OpenRouter slug | Pricing ($/M in / out) | +|---|---|---| +| Claude Opus 4.8 | `anthropic/claude-opus-4.8` | (see openrouter.ai) | +| Claude Sonnet 4.6 | `anthropic/claude-sonnet-4.6` | | +| Claude Haiku 4.5 | `anthropic/claude-haiku-4.5` | | +| GPT-5.5 (xhigh) | `openai/gpt-5.5` + `reasoning.effort=xhigh` | 5.00 / 30.00 | +| GPT-5.5 (high) | `openai/gpt-5.5` + `reasoning.effort=high` | 5.00 / 30.00 | +| GPT-5.5 (medium) | `openai/gpt-5.5` + `reasoning.effort=medium` | 5.00 / 30.00 | +| GPT-5.5 (low) | `openai/gpt-5.5` + `reasoning.effort=low` | 5.00 / 30.00 | +| Gemini 3.5 Flash | `google/gemini-3.5-flash` | 1.50 / 9.00 | +| Gemini 3.1 Pro | `google/gemini-3.1-pro-preview` | 2.00 / 12.00 | +| Nemotron 3 Ultra | `nvidia/nemotron-3-ultra-550b-a55b` | 0.50 / 2.50 | +| GPT-OSS 120B | `openai/gpt-oss-120b` | ~0.10 / 0.45 | + +Notes: +- Gemini 3.1 Pro only exists as a `-preview` slug; re-verify before the run. +- The four GPT-5.5 effort rows depend on H-5 (reasoning passthrough). +- Open-weight rows (Nemotron, GPT-OSS) chosen for brand recognition with US + educators; Llama and Mistral dropped per product decision 2026-06-10. + +## Run protocol for the official leaderboard (after fixes) + +```bash +python -m scripts.run_models \ + --models \ + --pack operations --runs 5 --judge --out results/operations +# repeat for outcomes and equity_research, then merge per-pack results +``` + +- `--runs 5` (never 1 — see H-3), temperature 1.0 (default), judge required. +- Judge is auto-selected per candidate (Opus 4.8 default; GPT-5.5 xhigh for + all Claude-family candidates — no judge shares a family with its + candidate). `--judge-model` overrides. +- Record judge model + version in the methodology page alongside results, + including the per-candidate judge assignment and the calibration caveat + noted under C-3. From 9ec15c13148f3ed61b242cd2684e29591c63cb4a Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Wed, 10 Jun 2026 16:19:24 -0400 Subject: [PATCH 2/5] fix(adapter): correct model registry to live OpenRouter slugs + reasoning-effort passthrough MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEED_MODELS mapped shorthands to stale placeholder slugs — "gpt-5" actually called openai/gpt-4o, "gemini-2.5-pro" called google/gemini-pro-1.5, and "claude-opus-4-7" called anthropic/claude-opus-4.5. A leaderboard published from these would attribute GPT-4o scores to GPT-5 (methodology review C-1). The adapter default also used a dash-form slug (claude-sonnet-4-5) that does not exist on OpenRouter — live slugs use dots. - Replace SEED_MODELS with the launch lineup, every slug verified against the live /api/v1/models endpoint on 2026-06-10: Claude Opus 4.8 / Sonnet 4.6 / Haiku 4.5, GPT-5.5, GPT-OSS 120B, Gemini 3.5 Flash, Gemini 3.1 Pro (preview), Nemotron 3 Ultra - Default adapter model -> anthropic/claude-sonnet-4.6 - post_chat_completion gains an optional reasoning_effort parameter (forwarded as {"reasoning": {"effort": ...}}) for reasoning-capable models; needed by the GPT-5.5 judge and the candidate-side effort sweep - Update stale slugs in README, quickstart, and adapter docstrings --- README.md | 2 +- examples/quickstart.md | 4 +- runner/adapters/__init__.py | 4 +- runner/adapters/openrouter_adapter.py | 65 ++++++++++++++++----------- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 65a04a7..0b55551 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ export OPENROUTER_API_KEY="sk-or-..." python -m runner.cli \ --pack operations \ --adapter openrouter \ - --model anthropic/claude-sonnet-4-5 \ + --model anthropic/claude-sonnet-4.6 \ --runs 5 \ --out /tmp/grade_real ``` diff --git a/examples/quickstart.md b/examples/quickstart.md index 4bff365..83d2452 100644 --- a/examples/quickstart.md +++ b/examples/quickstart.md @@ -95,13 +95,13 @@ export OPENROUTER_API_KEY="sk-or-..." python -m runner.cli \ --pack operations \ --adapter openrouter \ - --model anthropic/claude-sonnet-4-5 \ + --model anthropic/claude-sonnet-4.6 \ --runs 5 \ --out /tmp/grade_real_out ``` The `--model` flag accepts any model slug available on OpenRouter, for -example `openai/gpt-4o` or `anthropic/claude-opus-4-7`. +example `openai/gpt-5.5` or `anthropic/claude-opus-4.8`. --- diff --git a/runner/adapters/__init__.py b/runner/adapters/__init__.py index 4cbcb1f..d7f7058 100644 --- a/runner/adapters/__init__.py +++ b/runner/adapters/__init__.py @@ -24,7 +24,7 @@ # Resolve by name (returns a class, not an instance): AdapterCls = get_adapter("openrouter") - adapter = AdapterCls(model="anthropic/claude-sonnet-4-5") + adapter = AdapterCls(model="anthropic/claude-sonnet-4.6") # Use the default adapter: DefaultCls = get_adapter(DEFAULT_ADAPTER) @@ -61,7 +61,7 @@ def get_adapter(name: str) -> type: Example:: AdapterCls = get_adapter("openrouter") - adapter = AdapterCls(model="openai/gpt-4o") + adapter = AdapterCls(model="openai/gpt-5.5") """ if name not in ADAPTER_REGISTRY: known = ", ".join(sorted(ADAPTER_REGISTRY)) diff --git a/runner/adapters/openrouter_adapter.py b/runner/adapters/openrouter_adapter.py index d07851d..9e0cac9 100644 --- a/runner/adapters/openrouter_adapter.py +++ b/runner/adapters/openrouter_adapter.py @@ -10,21 +10,17 @@ Seed model identifiers ---------------------- -The mapping below lists the five seed models used in the GRADE arena (F4). -Pass the *key* as ``--model`` on the CLI; the adapter will resolve it to the -correct OpenRouter slug automatically. +The mapping below lists the launch leaderboard models. Pass the *key* as +``--model`` on the CLI; the adapter will resolve it to the correct OpenRouter +slug automatically. Every slug is verified against the live OpenRouter +``/api/v1/models`` listing — see ``docs/methodology_review_findings.md`` for +the verification date and full lineup rationale. .. code-block:: python from runner.adapters.openrouter_adapter import SEED_MODELS - print(SEED_MODELS) - # { - # "claude-opus-4-7": "anthropic/claude-opus-4-5", - # "claude-sonnet-4-6": "anthropic/claude-sonnet-4-5", - # "claude-haiku-4-5": "anthropic/claude-haiku-4-5", - # "gpt-5": "openai/gpt-4o", - # "gemini-2.5-pro": "google/gemini-pro-1.5", - # } + print(SEED_MODELS["claude-sonnet-4-6"]) + # "anthropic/claude-sonnet-4.6" Usage ----- @@ -33,7 +29,7 @@ import os from runner.adapters.openrouter_adapter import OpenRouterAdapter - adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4-5") + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6") output = adapter.run(task, run_index=0) The adapter reads ``OPENROUTER_API_KEY`` from the environment if *api_key* is @@ -100,20 +96,27 @@ def _get_openrouter_timeout() -> float: #: Seed model shorthand → OpenRouter model slug mapping. #: -#: These are the five models seeded in the GRADE arena (ticket F4). Pass a -#: shorthand string as the ``model`` constructor argument and the adapter -#: resolves it automatically; you may also supply any raw OpenRouter slug -#: directly (e.g. ``"anthropic/claude-3-5-sonnet"``) and it will be used -#: verbatim. +#: These are the launch leaderboard models. Pass a shorthand string as the +#: ``model`` constructor argument and the adapter resolves it automatically; +#: you may also supply any raw OpenRouter slug directly (e.g. +#: ``"anthropic/claude-sonnet-4.6"``) and it will be used verbatim. +#: +#: Every slug below was verified against the live OpenRouter +#: ``/api/v1/models`` listing on 2026-06-10. Note that current OpenRouter +#: slugs use dots in version numbers (``claude-opus-4.8``), not dashes. SEED_MODELS: dict[str, str] = { - # Claude family (via Anthropic on OpenRouter) - "claude-opus-4-7": "anthropic/claude-opus-4-5", - "claude-sonnet-4-6": "anthropic/claude-sonnet-4-5", - "claude-haiku-4-5": "anthropic/claude-haiku-4-5", + # Anthropic + "claude-opus-4-8": "anthropic/claude-opus-4.8", + "claude-sonnet-4-6": "anthropic/claude-sonnet-4.6", + "claude-haiku-4-5": "anthropic/claude-haiku-4.5", # OpenAI - "gpt-5": "openai/gpt-4o", + "gpt-5-5": "openai/gpt-5.5", + "gpt-oss-120b": "openai/gpt-oss-120b", # Google - "gemini-2.5-pro": "google/gemini-pro-1.5", + "gemini-3-5-flash": "google/gemini-3.5-flash", + "gemini-3-1-pro": "google/gemini-3.1-pro-preview", + # NVIDIA + "nemotron-3-ultra": "nvidia/nemotron-3-ultra-550b-a55b", } @@ -381,7 +384,7 @@ class OpenRouterAdapter: Args: model: OpenRouter model slug or :data:`SEED_MODELS` shorthand. - Defaults to ``"anthropic/claude-sonnet-4-5"`` (Sonnet 4.6 on + Defaults to ``"anthropic/claude-sonnet-4.6"`` (Sonnet 4.6 on OpenRouter). api_key: OpenRouter API key. If ``None``, falls back to the ``OPENROUTER_API_KEY`` environment variable. Raises @@ -410,7 +413,7 @@ class OpenRouterAdapter: def __init__( self, - model: str = "anthropic/claude-sonnet-4-5", + model: str = "anthropic/claude-sonnet-4.6", api_key: str | None = None, temperature: float = 1.0, max_tokens: int | None = None, @@ -635,6 +638,7 @@ def post_chat_completion( max_tokens: int = ..., api_key: str | None = ..., return_usage: Literal[False] = ..., + reasoning_effort: str | None = ..., ) -> str: ... @@ -647,6 +651,7 @@ def post_chat_completion( api_key: str | None = ..., *, return_usage: Literal[True], + reasoning_effort: str | None = ..., ) -> tuple[str, ChatCompletionUsage]: ... @@ -658,6 +663,7 @@ def post_chat_completion( max_tokens: int = ..., api_key: str | None = ..., return_usage: bool = ..., + reasoning_effort: str | None = ..., ) -> str | tuple[str, ChatCompletionUsage]: ... @@ -668,6 +674,7 @@ def post_chat_completion( max_tokens: int = 16, api_key: str | None = None, return_usage: bool = False, + reasoning_effort: str | None = None, ) -> str | tuple[str, ChatCompletionUsage]: """Send a chat-completion request to OpenRouter and return the response text. @@ -690,6 +697,12 @@ def post_chat_completion( return_usage: When ``True``, return a ``(text, ChatCompletionUsage)`` tuple instead of just the response text. Existing callers that do not pass this flag are unaffected (backward-compatible). + reasoning_effort: Optional reasoning effort level (e.g. ``"low"``, + ``"medium"``, ``"high"``, ``"xhigh"``) forwarded to OpenRouter as + ``{"reasoning": {"effort": ...}}``. Only meaningful for + reasoning-capable models; ``None`` (default) omits the field. + NOTE: for reasoning models, *max_tokens* bounds reasoning + + visible output combined, so callers must budget accordingly. Returns: When *return_usage* is ``False`` (default): the model's response text. @@ -723,6 +736,8 @@ def post_chat_completion( "temperature": temperature, "max_tokens": max_tokens, } + if reasoning_effort is not None: + payload["reasoning"] = {"effort": reasoning_effort} timeout = _get_openrouter_timeout() From 1569186f0f14744608fb6074ec9455d2f2306c5a Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Wed, 10 Jun 2026 16:20:14 -0400 Subject: [PATCH 3/5] fix(judge): cross-family judge policy, rationale-then-score, no silent failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes out methodology review findings C-2, C-3, and M-6 — the judge could silently produce meaningless or biased scores in three ways. Null-judge guard (C-2): without --judge, the C2-owned dimensions (40% of the composite) were a flat 0.5 placeholder for every model with nothing in the result recording it. - dispatcher stamps a "null_judge" flag into every task's scorer_flags when no live judge is configured - runner.cli and scripts.run_models refuse the openrouter adapter without --judge unless --allow-null-judge is passed (stub/smoke runs unaffected) - a judge-client construction failure in the batch runner now records the model as failed instead of silently downgrading to the null judge Judge selection policy (C-3): no judge ever shares a model family with its candidate, so house style cannot bias the judged dimensions. Claude Opus 4.8 (strongest available) judges every candidate except Claude-family models, which GPT-5.5 at xhigh reasoning effort judges instead. Implemented in select_judge_model(); both CLIs apply it per candidate and print the judge in use; --judge-model overrides. The old default judge slug (anthropic/claude-opus-4-5, dash form) did not exist on OpenRouter — every live judge call would have failed. Rationale-then-score (M-6): the judge now justifies in 2-3 sentences and ends with a "SCORE: " line instead of replying with a bare float. Parsing prefers the last SCORE: line and falls back to the last float, so numbers quoted in the justification are never mistaken for the verdict. A non-parseable reply retries once, then raises JudgeScoreError — surfacing in failures.json instead of silently scoring the candidate 0.0. Non-reasoning judge max_tokens 16 -> 384; reasoning judges get 8192 (reasoning tokens share the budget with the visible answer). --- benchmark/rubrics/judge_client.py | 220 +++++++++++++++++++++----- runner/cli.py | 56 ++++++- runner/dispatcher.py | 8 + scripts/run_models.py | 62 +++++++- tests/unit/test_cli_judge.py | 14 +- tests/unit/test_openrouter_adapter.py | 121 +++++++++++++- 6 files changed, 430 insertions(+), 51 deletions(-) diff --git a/benchmark/rubrics/judge_client.py b/benchmark/rubrics/judge_client.py index 3c9a665..2c8e850 100644 --- a/benchmark/rubrics/judge_client.py +++ b/benchmark/rubrics/judge_client.py @@ -33,7 +33,10 @@ insights, required limitations, forbidden claims). 4. The full model output text (``raw_response_text`` if present, else the ``key_findings`` and ``limitations`` lists joined as prose). -5. An instruction to respond with a single float in [0, 1]. +5. An instruction to give a brief (2-3 sentence) justification, then a + final ``SCORE: `` line with the score in [0, 1] + (rationale-then-score is measurably more reliable than bare-float + judging). The judge model is queried at **temperature 0** so that results are reproducible across runs. @@ -42,9 +45,11 @@ from __future__ import annotations import json +import re from typing import Any -#: Number of judge calls permitted per task invocation (one per dimension). +#: Upper bound of judge calls per score_rubric invocation (one per scored +#: dimension; the runner judges only the three C2-owned dimensions). MAX_JUDGE_CALLS_PER_TASK: int = 6 #: Task fields that contain raw fixture data or internal dispatcher keys. @@ -64,7 +69,108 @@ JUDGE_SEED: int = 42 #: Default judge model identifier (OpenRouter slug). -DEFAULT_JUDGE_MODEL: str = "anthropic/claude-opus-4-5" +#: +#: Judge-selection policy (decided 2026-06-10, see +#: ``docs/methodology_review_findings.md`` C-3): the strongest available +#: model judges everyone — Claude Opus 4.8 — EXCEPT when the candidate under +#: test is itself a Claude-family model, in which case GPT-5.5 at xhigh +#: reasoning effort judges instead. No judge ever shares a model family with +#: its candidate, so house style cannot bias the judged dimensions. Use +#: :func:`select_judge_model` to apply this policy; ``--judge-model`` on both +#: CLIs overrides it. +#: +#: NOTE: current OpenRouter slugs use dots in version numbers +#: (``claude-opus-4.8``), not dashes — the dash form is rejected by the API. +DEFAULT_JUDGE_MODEL: str = "anthropic/claude-opus-4.8" + +#: Judge used when the candidate model is a Claude-family model (so a Claude +#: judge never grades a Claude candidate). GPT-5.5 is a reasoning model, so +#: judge calls for it must send a reasoning-effort hint and a much larger +#: token budget. +CLAUDE_CANDIDATE_JUDGE_MODEL: str = "openai/gpt-5.5" + +#: Reasoning effort used for the :data:`CLAUDE_CANDIDATE_JUDGE_MODEL` judge. +CLAUDE_CANDIDATE_JUDGE_REASONING_EFFORT: str = "xhigh" + +#: max_tokens for reasoning-model judges. Reasoning tokens and the visible +#: justification + SCORE line share this budget, so it must be large. +REASONING_JUDGE_MAX_TOKENS: int = 8192 + +#: Substrings identifying a Claude-family candidate (checked lowercase). +_CLAUDE_FAMILY_MARKERS: tuple[str, ...] = ("claude", "anthropic/", "opus", "sonnet", "haiku") + + +def select_judge_model(candidate_model_id: str | None) -> tuple[str, str | None]: + """Pick the judge for *candidate_model_id* per the cross-family policy. + + The strongest model (Claude Opus 4.8) judges every candidate, except + Claude-family candidates — those are judged by GPT-5.5 at xhigh reasoning + effort, so no judge ever shares a family (and hence a house style) with + the model it grades. + + Args: + candidate_model_id: The model under test (shorthand or OpenRouter + slug, e.g. ``"claude-opus-4-8"`` or ``"anthropic/claude-opus-4.8"``). + ``None`` or empty selects the default judge. + + Returns: + A ``(judge_model_slug, reasoning_effort)`` tuple ready to construct a + :class:`JudgeClient`. *reasoning_effort* is ``None`` for + non-reasoning judges. + """ + candidate = (candidate_model_id or "").lower() + if any(marker in candidate for marker in _CLAUDE_FAMILY_MARKERS): + return CLAUDE_CANDIDATE_JUDGE_MODEL, CLAUDE_CANDIDATE_JUDGE_REASONING_EFFORT + return DEFAULT_JUDGE_MODEL, None + + +class JudgeScoreError(RuntimeError): + """Raised when the judge model's response cannot be parsed as a score. + + Raised only after the configured retry has also failed. Surfacing this + as an exception (rather than silently scoring 0.0) ensures a judge-side + failure is recorded as a task failure in ``failures.json`` instead of + being charged against the candidate model's score. + """ + + +#: Regex that extracts a float (e.g. ``0.85``, ``1``, ``.5``) from a judge +#: reply, tolerating stray punctuation, markdown, or label prefixes. +_FLOAT_PATTERN = re.compile(r"-?(?:\d+\.?\d*|\.\d+)") + +#: Regex for the canonical final ``SCORE: `` line requested by the +#: judge prompt. Matched case-insensitively and tolerant of markdown +#: emphasis around the label (e.g. ``**SCORE: 0.85**``). +_SCORE_LINE_PATTERN = re.compile(r"SCORE\s*[:=]\s*(-?(?:\d+\.?\d*|\.\d+))", re.IGNORECASE) + +#: Default max_tokens for non-reasoning judges. The rationale-then-score +#: protocol needs room for 2-3 sentences of justification plus the final +#: ``SCORE:`` line. +JUDGE_MAX_TOKENS: int = 384 + + +def _parse_judge_score(raw: str) -> float | None: + """Extract the judge's score from *raw* response text. + + Prefers the last ``SCORE: `` line (the prompt asks for it as the + final line; "last" guards against the justification quoting the format). + Falls back to the last bare float in the text — under rationale-then- + score, trailing numbers are far more likely to be the verdict than + numbers quoted from the response being judged. + + Args: + raw: Stripped judge response text. + + Returns: + The parsed score, or ``None`` if no float is present at all. + """ + score_matches = _SCORE_LINE_PATTERN.findall(raw) + if score_matches: + return float(score_matches[-1]) + float_matches = _FLOAT_PATTERN.findall(raw) + if float_matches: + return float(float_matches[-1]) + return None def _build_judge_prompt( @@ -161,9 +267,13 @@ def _build_judge_prompt( "", "## Instruction", f"Score the model response on the '{dimension}' dimension only.", - "Return a single floating-point number between 0.0 and 1.0 (inclusive),", - "where 0.0 = completely fails the dimension and 1.0 = fully satisfies it.", - "Respond with ONLY the number — no explanation, no punctuation, no units.", + "First, briefly justify your assessment in 2-3 sentences, referring to", + "the reference material above.", + "Then, on the final line, output exactly:", + "SCORE: ", + "where is a floating-point number between 0.0 and 1.0", + "(inclusive), 0.0 = completely fails the dimension and 1.0 = fully", + "satisfies it.", ] return "\n".join(lines) @@ -198,20 +308,31 @@ class JudgeClient: model: OpenRouter model slug to use as judge. Defaults to :data:`DEFAULT_JUDGE_MODEL`. max_tokens: Maximum tokens to request in the judge response. Defaults - to 16 (a single float token). + to :data:`JUDGE_MAX_TOKENS` (room for a short justification plus + the final ``SCORE:`` line), or + :data:`REASONING_JUDGE_MAX_TOKENS` when *reasoning_effort* is set + (reasoning tokens share the budget with the visible answer). + reasoning_effort: Optional reasoning effort (e.g. ``"xhigh"``) for + reasoning-model judges; forwarded to OpenRouter as + ``{"reasoning": {"effort": ...}}``. Example:: client = JudgeClient(api_key="sk-or-...") score = client.judge("grounding_accuracy", "", task, model_output) print(client.cumulative_cost_usd) + + # Per the no-self-judging policy: + judge_model, effort = select_judge_model("anthropic/claude-opus-4.8") + client = JudgeClient(model=judge_model, reasoning_effort=effort) """ def __init__( self, api_key: str | None = None, model: str = DEFAULT_JUDGE_MODEL, - max_tokens: int = 16, + max_tokens: int | None = None, + reasoning_effort: str | None = None, ) -> None: """Initialize the JudgeClient with OpenRouter settings. @@ -219,11 +340,22 @@ def __init__( api_key: OpenRouter API key. If *None*, reads from the ``OPENROUTER_API_KEY`` environment variable at call time. model: OpenRouter model slug for the judge. - max_tokens: Maximum tokens for the judge response. + max_tokens: Maximum tokens for the judge response. ``None`` + (default) resolves to :data:`JUDGE_MAX_TOKENS`, or + :data:`REASONING_JUDGE_MAX_TOKENS` when *reasoning_effort* + is set. + reasoning_effort: Optional reasoning effort level for + reasoning-model judges (``None`` omits the field). """ self._api_key = api_key self._model = model - self._max_tokens = max_tokens + self._reasoning_effort = reasoning_effort + if max_tokens is not None: + self._max_tokens = max_tokens + elif reasoning_effort is not None: + self._max_tokens = REASONING_JUDGE_MAX_TOKENS + else: + self._max_tokens = JUDGE_MAX_TOKENS # Accumulated judge-cost state — updated after every judge() call. #: Total cost in USD across all judge() calls; None until the provider @@ -259,36 +391,50 @@ def judge( Returns: A float in ``[0.0, 1.0]`` parsed from the judge model's response. - If parsing fails, returns ``0.0`` and the failure is silent — - callers should log at a higher level if desired. + Parsing extracts the first float in the reply (tolerating stray + punctuation or markdown). If no float can be extracted, the API + call is retried once; if the retry also fails, + :exc:`JudgeScoreError` is raised so the failure surfaces as a + task failure rather than silently scoring the candidate 0.0. + + Raises: + JudgeScoreError: If the judge response contains no parseable + float after one retry. """ # Lazy import so this module is importable without httpx installed. from runner.adapters.openrouter_adapter import post_chat_completion prompt = _build_judge_prompt(dimension, guidance, task, model_output) - result = post_chat_completion( - messages=[{"role": "user", "content": prompt}], - model=self._model, - temperature=0.0, - max_tokens=self._max_tokens, - api_key=self._api_key, - return_usage=True, + + last_raw: str = "" + for _attempt in range(2): # initial call + one retry on parse failure + result = post_chat_completion( + messages=[{"role": "user", "content": prompt}], + model=self._model, + temperature=0.0, + max_tokens=self._max_tokens, + api_key=self._api_key, + return_usage=True, + reasoning_effort=self._reasoning_effort, + ) + # post_chat_completion with return_usage=True always returns a tuple. + raw, usage = result + last_raw = raw.strip() + + # Accumulate usage into instance state. + self.judge_call_count += 1 + if usage.cost_usd is not None: + self.cumulative_cost_usd = (self.cumulative_cost_usd or 0.0) + usage.cost_usd + if usage.prompt_tokens is not None: + self.cumulative_prompt_tokens += usage.prompt_tokens + if usage.completion_tokens is not None: + self.cumulative_completion_tokens += usage.completion_tokens + + score = _parse_judge_score(last_raw) + if score is not None: + return max(0.0, min(1.0, score)) + + raise JudgeScoreError( + f"Judge model {self._model!r} returned no parseable score for " + f"dimension {dimension!r} after retry. Last response: {last_raw!r}" ) - # post_chat_completion with return_usage=True always returns a tuple. - raw, usage = result - raw = raw.strip() - - # Accumulate usage into instance state. - self.judge_call_count += 1 - if usage.cost_usd is not None: - self.cumulative_cost_usd = (self.cumulative_cost_usd or 0.0) + usage.cost_usd - if usage.prompt_tokens is not None: - self.cumulative_prompt_tokens += usage.prompt_tokens - if usage.completion_tokens is not None: - self.cumulative_completion_tokens += usage.completion_tokens - - try: - score = float(raw) - except ValueError: - score = 0.0 - return max(0.0, min(1.0, score)) diff --git a/runner/cli.py b/runner/cli.py index daecc65..a92699e 100644 --- a/runner/cli.py +++ b/runner/cli.py @@ -41,7 +41,7 @@ # With the OpenRouter adapter: python -m runner.cli --pack outcomes --adapter openrouter \ - --model anthropic/claude-sonnet-4-5 --runs 5 --out /tmp/grade_out + --model anthropic/claude-sonnet-4.6 --runs 5 --out /tmp/grade_out """ from __future__ import annotations @@ -197,7 +197,7 @@ def build_parser() -> argparse.ArgumentParser: metavar="MODEL", help=( "Model slug or shorthand forwarded to the adapter " - "(e.g. 'anthropic/claude-sonnet-4-5', 'claude-sonnet-4-6'). " + "(e.g. 'anthropic/claude-sonnet-4.6', 'claude-sonnet-4-6'). " "Required for the openrouter adapter; ignored by the stub." ), ) @@ -252,7 +252,28 @@ def build_parser() -> argparse.ArgumentParser: help=( "Use a live OpenRouter-backed judge for the rubric (C2) and claim-validation " "(C3) judge fallback, instead of the default null judge (0.5). Requires " - "OPENROUTER_API_KEY." + "OPENROUTER_API_KEY. Required for the openrouter adapter unless " + "--allow-null-judge is passed." + ), + ) + parser.add_argument( + "--judge-model", + default=None, + metavar="SLUG", + help=( + "OpenRouter slug for the judge model (default: " + "benchmark.rubrics.judge_client.DEFAULT_JUDGE_MODEL). " + "Only meaningful together with --judge." + ), + ) + parser.add_argument( + "--allow-null-judge", + action="store_true", + help=( + "Explicitly allow running the openrouter adapter without a live judge. " + "The C2-owned dimensions (40%% of the composite) are then a flat 0.5 " + "placeholder and every task is flagged 'null_judge' in the result. " + "Never use this for leaderboard runs." ), ) return parser @@ -300,11 +321,36 @@ def main(argv: list[str] | None = None) -> int: out_dir = Path(args.out) # --- Build judge (optional, live OpenRouter) --- + # Guard against silently producing placeholder scorecards: without a live + # judge, 40% of the composite is a flat 0.5 for every model. + if args.adapter == "openrouter" and not args.judge and not args.allow_null_judge: + print( + "ERROR: the openrouter adapter requires --judge (live rubric scoring). " + "Without it, insight_quality/evidence_linkage/structure_usability are a " + "flat 0.5 placeholder. Pass --allow-null-judge to override for smoke " + "tests only.", + file=sys.stderr, + ) + return 1 + + # Judge selection follows the no-self-judging policy in + # benchmark.rubrics.judge_client.select_judge_model: Opus 4.8 judges + # everyone except Claude-family candidates, which GPT-5.5 (xhigh) judges + # so no judge shares a family with its candidate. + # --judge-model overrides the policy. judge_client: object | None = None if args.judge: - from benchmark.rubrics.judge_client import JudgeClient + from benchmark.rubrics.judge_client import JudgeClient, select_judge_model - judge_client = JudgeClient() + if args.judge_model: + judge_model, judge_effort = args.judge_model, None + else: + judge_model, judge_effort = select_judge_model(args.model_id or args.model) + print( + f"Judge: {judge_model}" + + (f" (reasoning effort: {judge_effort})" if judge_effort else "") + ) + judge_client = JudgeClient(model=judge_model, reasoning_effort=judge_effort) # --- Run pack via shared helper --- try: diff --git a/runner/dispatcher.py b/runner/dispatcher.py index 36d755e..109c29d 100644 --- a/runner/dispatcher.py +++ b/runner/dispatcher.py @@ -375,6 +375,14 @@ def run_task( per_run_scores: list[DimensionScores] = [] all_flags: list[str] = [] + # Make null-judge runs visible in the result: when no live judge is + # configured, the C2-owned dimensions (insight_quality, evidence_linkage, + # structure_usability — 40% of the composite) are a flat 0.5 placeholder. + # The flag lands in per_task_scores[].scorer_flags so downstream consumers + # (leaderboard, website) can detect and refuse placeholder scorecards. + if judge_client is None: + all_flags.append("null_judge") + # Resolve fixture contents once (same for all runs of this task). fixtures: dict[str, str] = _resolve_fixtures( task.get("allowed_inputs", []), diff --git a/scripts/run_models.py b/scripts/run_models.py index be5e0ec..afaa0a2 100644 --- a/scripts/run_models.py +++ b/scripts/run_models.py @@ -222,7 +222,28 @@ def build_parser() -> argparse.ArgumentParser: help=( "Use a live OpenRouter-backed judge (C2/C3 rubric + claim validation). " "A fresh JudgeClient is constructed per model so judge costs are isolated. " - "Requires OPENROUTER_API_KEY." + "Requires OPENROUTER_API_KEY. Required for the openrouter adapter " + "unless --allow-null-judge is passed." + ), + ) + parser.add_argument( + "--judge-model", + default=None, + metavar="SLUG", + help=( + "OpenRouter slug for the judge model (default: " + "benchmark.rubrics.judge_client.DEFAULT_JUDGE_MODEL). " + "Only meaningful together with --judge." + ), + ) + parser.add_argument( + "--allow-null-judge", + action="store_true", + help=( + "Explicitly allow running the openrouter adapter without a live judge. " + "The C2-owned dimensions (40%% of the composite) are then a flat 0.5 " + "placeholder and every task is flagged 'null_judge' in the result. " + "Never use this for leaderboard runs." ), ) parser.add_argument( @@ -275,6 +296,18 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: {exc}", file=sys.stderr) return 1 + # Guard against silently producing placeholder leaderboards: without a + # live judge, 40% of every composite is a flat 0.5. + if args.adapter == "openrouter" and not args.judge and not args.allow_null_judge: + print( + "ERROR: the openrouter adapter requires --judge (live rubric scoring). " + "Without it, insight_quality/evidence_linkage/structure_usability are a " + "flat 0.5 placeholder for every model and the leaderboard is meaningless. " + "Pass --allow-null-judge to override for smoke tests only.", + file=sys.stderr, + ) + return 1 + out_root = Path(args.out) out_root.mkdir(parents=True, exist_ok=True) @@ -299,14 +332,33 @@ def main(argv: list[str] | None = None) -> int: continue # Build a fresh JudgeClient per model (isolates judge cost). + # Judge selection follows the no-self-judging policy in + # benchmark.rubrics.judge_client.select_judge_model: Opus 4.8 judges + # everyone except Claude-family candidates, which GPT-5.5 (xhigh) + # judges so no judge shares a family with its candidate. + # --judge-model overrides the policy for the whole batch. + # A judge construction failure must NOT silently downgrade to the + # null judge — that would score this model on placeholder 0.5s while + # other models get real judge scores. Record it as a model failure. judge_client: object | None = None if args.judge: try: - from benchmark.rubrics.judge_client import JudgeClient - - judge_client = JudgeClient() + from benchmark.rubrics.judge_client import JudgeClient, select_judge_model + + if args.judge_model: + judge_model, judge_effort = args.judge_model, None + else: + judge_model, judge_effort = select_judge_model(model_id) + print( + f" judge: {judge_model}" + + (f" (reasoning effort: {judge_effort})" if judge_effort else ""), + flush=True, + ) + judge_client = JudgeClient(model=judge_model, reasoning_effort=judge_effort) except Exception as exc: # noqa: BLE001 - print(f" WARNING: failed to build judge client: {exc}", flush=True) + print(f" FAILED to build judge client: {exc}", flush=True) + failed_models.append({"model_id": model_id, "error": f"judge client: {exc}"}) + continue try: result = run_pack( diff --git a/tests/unit/test_cli_judge.py b/tests/unit/test_cli_judge.py index d36ce6a..d9f5a96 100644 --- a/tests/unit/test_cli_judge.py +++ b/tests/unit/test_cli_judge.py @@ -33,8 +33,13 @@ def fake_run_pack(**kwargs: Any) -> dict[str, Any]: raise RuntimeError("stop-after-capture") # cli.main catches -> returns 1 sentinel = object() + + def fake_judge_client(**kwargs: Any) -> object: + captured["judge_kwargs"] = kwargs + return sentinel + monkeypatch.setattr(cli, "run_pack", fake_run_pack) - monkeypatch.setattr("benchmark.rubrics.judge_client.JudgeClient", lambda: sentinel) + monkeypatch.setattr("benchmark.rubrics.judge_client.JudgeClient", fake_judge_client) rc = cli.main( [ @@ -51,6 +56,13 @@ def fake_run_pack(**kwargs: Any) -> dict[str, Any]: ) assert rc == 1 assert captured["judge_client"] is sentinel + # No --judge-model and a non-Opus (stub) candidate → policy default judge. + from benchmark.rubrics.judge_client import DEFAULT_JUDGE_MODEL + + judge_kwargs = captured["judge_kwargs"] + assert isinstance(judge_kwargs, dict) + assert judge_kwargs["model"] == DEFAULT_JUDGE_MODEL + assert judge_kwargs["reasoning_effort"] is None def test_main_uses_null_judge_without_flag(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: diff --git a/tests/unit/test_openrouter_adapter.py b/tests/unit/test_openrouter_adapter.py index 1f993db..5bf8d6a 100644 --- a/tests/unit/test_openrouter_adapter.py +++ b/tests/unit/test_openrouter_adapter.py @@ -623,15 +623,52 @@ def test_judge_clamps_below_zero(self) -> None: score = client.judge("grounding_accuracy", "", _VALID_TASK, self._VALID_OUTPUT) assert score == pytest.approx(0.0) - def test_judge_non_float_response_falls_back_to_zero(self) -> None: - """Non-parseable response must fall back to 0.0 without raising.""" + def test_judge_non_float_response_raises_after_retry(self) -> None: + """A non-parseable response must retry once, then raise JudgeScoreError. + + Silently scoring 0.0 would charge a judge-side failure against the + candidate model (methodology review finding C-3). + """ mock_body = {"choices": [{"message": {"role": "assistant", "content": "I am unsure."}}]} + with _MockHttpx(mock_body) as mock_post: + from benchmark.rubrics.judge_client import JudgeClient, JudgeScoreError + + client = JudgeClient(api_key="sk-or-test") + with pytest.raises(JudgeScoreError): + client.judge("grounding_accuracy", "", _VALID_TASK, self._VALID_OUTPUT) + assert mock_post.call_count == 2, "judge must retry exactly once on parse failure" + + def test_judge_extracts_float_from_noisy_response(self) -> None: + """A score embedded in prose/markdown must be extracted, not zeroed.""" + mock_body = {"choices": [{"message": {"role": "assistant", "content": "**Score: 0.85**"}}]} with _MockHttpx(mock_body): from benchmark.rubrics.judge_client import JudgeClient client = JudgeClient(api_key="sk-or-test") score = client.judge("grounding_accuracy", "", _VALID_TASK, self._VALID_OUTPUT) - assert score == pytest.approx(0.0) + assert score == pytest.approx(0.85) + + def test_judge_prefers_score_line_over_rationale_numbers(self) -> None: + """Numbers quoted in the justification must not be mistaken for the score.""" + rationale = ( + "The response cites 135 students and an 82.4% attendance rate, " + "both consistent with the gold facts.\nSCORE: 0.9" + ) + mock_body = {"choices": [{"message": {"role": "assistant", "content": rationale}}]} + with _MockHttpx(mock_body): + from benchmark.rubrics.judge_client import JudgeClient + + client = JudgeClient(api_key="sk-or-test") + score = client.judge("grounding_accuracy", "", _VALID_TASK, self._VALID_OUTPUT) + assert score == pytest.approx(0.9) + + def test_judge_prompt_requests_rationale_then_score(self) -> None: + """The judge prompt must ask for justification and a final SCORE line.""" + from benchmark.rubrics.judge_client import _build_judge_prompt + + prompt = _build_judge_prompt("insight_quality", "", _VALID_TASK, self._VALID_OUTPUT) + assert "SCORE:" in prompt + assert "justify" in prompt.lower() or "justification" in prompt.lower() def test_judge_uses_openrouter_not_anthropic(self) -> None: """JudgeClient must call OpenRouter endpoint, not Anthropic directly.""" @@ -668,6 +705,84 @@ def test_judge_satisfies_protocol(self) -> None: assert "model_output" in params +class TestJudgeSelection: + """Tests for the no-self-judging judge-selection policy.""" + + def test_non_claude_candidate_gets_opus_judge(self) -> None: + """Non-Claude candidates must be judged by the default Opus judge.""" + from benchmark.rubrics.judge_client import DEFAULT_JUDGE_MODEL, select_judge_model + + for candidate in ( + "openai/gpt-5.5", + "openai/gpt-oss-120b", + "google/gemini-3.5-flash", + "nvidia/nemotron-3-ultra-550b-a55b", + None, + "", + ): + judge, effort = select_judge_model(candidate) + assert judge == DEFAULT_JUDGE_MODEL, f"candidate {candidate!r}" + assert effort is None + + def test_claude_candidate_gets_gpt_judge(self) -> None: + """Claude-family candidates must never be judged by a Claude judge.""" + from benchmark.rubrics.judge_client import ( + CLAUDE_CANDIDATE_JUDGE_MODEL, + CLAUDE_CANDIDATE_JUDGE_REASONING_EFFORT, + select_judge_model, + ) + + for candidate in ( + "anthropic/claude-opus-4.8", + "claude-opus-4-8", + "anthropic/claude-sonnet-4.6", + "claude-sonnet-4-6", + "anthropic/claude-haiku-4.5", + "claude-haiku-4-5", + ): + judge, effort = select_judge_model(candidate) + assert judge == CLAUDE_CANDIDATE_JUDGE_MODEL, f"candidate {candidate!r}" + assert effort == CLAUDE_CANDIDATE_JUDGE_REASONING_EFFORT + assert "claude" not in judge.lower() + assert "anthropic" not in judge.lower() + + def test_reasoning_judge_sends_effort_and_large_budget(self) -> None: + """A reasoning-effort judge must send the reasoning field and a large max_tokens.""" + mock_body = {"choices": [{"message": {"role": "assistant", "content": "0.7"}}]} + with _MockHttpx(mock_body) as mock_post: + from benchmark.rubrics.judge_client import ( + REASONING_JUDGE_MAX_TOKENS, + JudgeClient, + ) + + client = JudgeClient( + api_key="sk-or-test", + model="openai/gpt-5.5", + reasoning_effort="xhigh", + ) + score = client.judge("insight_quality", "", _VALID_TASK, TestJudgeClient._VALID_OUTPUT) + + assert score == pytest.approx(0.7) + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert payload["reasoning"] == {"effort": "xhigh"} + assert payload["max_tokens"] == REASONING_JUDGE_MAX_TOKENS + + def test_default_judge_omits_reasoning_field(self) -> None: + """A non-reasoning judge must not send the reasoning field.""" + mock_body = {"choices": [{"message": {"role": "assistant", "content": "0.7"}}]} + with _MockHttpx(mock_body) as mock_post: + from benchmark.rubrics.judge_client import JudgeClient + + client = JudgeClient(api_key="sk-or-test") + client.judge("insight_quality", "", _VALID_TASK, TestJudgeClient._VALID_OUTPUT) + + from benchmark.rubrics.judge_client import JUDGE_MAX_TOKENS + + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert "reasoning" not in payload + assert payload["max_tokens"] == JUDGE_MAX_TOKENS + + # --------------------------------------------------------------------------- # Adapter registry / CLI resolution tests # --------------------------------------------------------------------------- From e77626b48dac23a7225241ed994206b500c2abcd Mon Sep 17 00:00:00 2001 From: jonathan bechtel Date: Wed, 10 Jun 2026 16:20:42 -0400 Subject: [PATCH 4/5] fix(scoring): pass limitations to C1 + judge only C2-owned dimensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes out methodology review findings M-1 and M-2. M-1: _score_c1_grounding never passed the output's limitations list to score_facts, even though the scorer documents it as a fallback search target. A gold-fact value stated only in a caveat ("only 977 students are reflected in this snapshot") was scored as not found. Now passed through. M-2: the C2 loop judged all six rubric dimensions per run, then three (grounding_accuracy, calibration_limitation_handling, consistency) were discarded and overridden by C1/C3/C4 — 390 wasted judge calls per model at 26 tasks x 5 runs. score_rubric gains a dimensions parameter (full-rubric weight validation unchanged; unknown names raise before any judge call) and the dispatcher requests only the three C2-owned dimensions, halving judge cost. Default score_rubric behavior (all six) is unchanged for existing callers. --- benchmark/rubrics/rubric_scoring.py | 38 ++++++++++++++++++++-------- runner/dispatcher.py | 36 ++++++++++++++++++-------- tests/unit/test_rubric_scoring.py | 23 +++++++++++++++++ tests/unit/test_runner_dispatcher.py | 37 +++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 21 deletions(-) diff --git a/benchmark/rubrics/rubric_scoring.py b/benchmark/rubrics/rubric_scoring.py index f354fcd..3f770e6 100644 --- a/benchmark/rubrics/rubric_scoring.py +++ b/benchmark/rubrics/rubric_scoring.py @@ -12,8 +12,11 @@ cannot express cross-field numeric constraints. Any rubric whose dimension weights do not sum to approximately 1.0 (within ``WEIGHT_SUM_TOLERANCE``) is rejected with :exc:`ValueError` before any judge calls are made. -- **Judge calls are capped** at one call per dimension (six total) per task, - providing a deterministic upper bound on API usage. +- **Judge calls are capped** at one call per scored dimension, providing a + deterministic upper bound on API usage. Callers may restrict scoring to a + subset of dimensions via the ``dimensions`` parameter; the runner judges + only the three C2-owned dimensions, since the other three are computed + deterministically by C1/C3/C4. - **Temperature 0 and a fixed random seed** are passed through to the judge client to ensure reproducible results; see :mod:`benchmark.rubrics.judge_client` for the live-judge integration wrapper. @@ -145,6 +148,7 @@ def score_rubric( task: dict[str, Any], model_output: dict[str, Any], judge_client: JudgeClientProtocol, + dimensions: tuple[str, ...] | None = None, ) -> dict[str, Any]: """Score *model_output* against *task*'s rubric using *judge_client*. @@ -168,21 +172,29 @@ def score_rubric( model_output: The normalized model output dict. Passed through to the judge client unchanged. judge_client: An object implementing :class:`JudgeClientProtocol`. The - scorer makes exactly one ``judge`` call per dimension (six calls - total) with no retries; cap and determinism settings (temperature 0, - fixed seed) should be configured on the client itself — see + scorer makes exactly one ``judge`` call per scored dimension; cap + and determinism settings (temperature 0, fixed seed) should be + configured on the client itself — see :mod:`benchmark.rubrics.judge_client`. + dimensions: Optional subset of :data:`RUBRIC_DIMENSIONS` to judge. + When ``None`` (default) all six dimensions are judged. The + dispatcher passes only the C2-owned dimensions + (``insight_quality``, ``evidence_linkage``, + ``structure_usability``) since the other three are owned by + C1/C3/C4 — judging them would be wasted API spend. Weight + validation always runs over the full rubric regardless. Returns: A dict with the following keys: - ``"task_id"`` (str): Copied from *task*. - ``"dimension_scores"`` (dict[str, float]): Per-dimension sub-scores - in [0, 1] keyed by dimension name. + in [0, 1] keyed by dimension name (scored dimensions only). - ``"rubric_weights"`` (dict[str, float]): The per-dimension weights - extracted from the task rubric. + extracted from the task rubric (scored dimensions only). - ``"composite"`` (float): The weighted aggregate score in [0, 1], - computed as ``sum(weight_i * score_i)`` over all six dimensions. + computed as ``sum(weight_i * score_i)`` over the scored dimensions + (partial when a ``dimensions`` subset is requested). Raises: ValueError: If the task rubric weights do not sum to @@ -202,14 +214,20 @@ def score_rubric( rubric: dict[str, Any] = task["rubric"] task_id: str = task["task_id"] - # Validate weight sum before making any judge calls. + # Validate weight sum before making any judge calls. Always validates + # the full rubric, even when only a subset of dimensions is judged. validate_rubric_weights(rubric) + scored_dimensions: tuple[str, ...] = dimensions if dimensions is not None else RUBRIC_DIMENSIONS + unknown = [d for d in scored_dimensions if d not in RUBRIC_DIMENSIONS] + if unknown: + raise ValueError(f"Unknown rubric dimension(s) requested: {unknown}") + dimension_scores: dict[str, float] = {} rubric_weights: dict[str, float] = {} composite: float = 0.0 - for dim in RUBRIC_DIMENSIONS: + for dim in scored_dimensions: dim_spec: dict[str, Any] = rubric[dim] weight: float = float(dim_spec["weight"]) guidance: str = dim_spec.get("guidance", "") diff --git a/runner/dispatcher.py b/runner/dispatcher.py index 109c29d..ebdbe14 100644 --- a/runner/dispatcher.py +++ b/runner/dispatcher.py @@ -214,6 +214,10 @@ def judge( def _score_c1_grounding(task: dict[str, Any], output: dict[str, Any]) -> float: """Run C1 fact scoring and return ``grounding_accuracy`` in [0, 1]. + The output's ``limitations`` list is passed through as a documented + fallback search target — models often state caveated numbers ("only 127 + of the 135 enrolled students attended") in their limitations section. + Args: task: Task definition dict. output: Normalized model output dict. @@ -225,10 +229,22 @@ def _score_c1_grounding(task: dict[str, Any], output: dict[str, Any]) -> float: gold_facts=task.get("gold_facts", []), structured_metrics=output.get("structured_metrics", {}), key_findings=output.get("key_findings", []), + limitations=output.get("limitations", []), ) return result.grounding_accuracy +#: The rubric dimensions whose authoritative score comes from the C2 judge. +#: The other three (grounding_accuracy, calibration_limitation_handling, +#: consistency) are owned by C1/C3/C4, so judging them would be wasted API +#: spend — at 26 tasks × 5 runs that's 390 discarded judge calls per model. +_C2_OWNED_DIMENSIONS: tuple[str, ...] = ( + "insight_quality", + "evidence_linkage", + "structure_usability", +) + + def _score_c2_rubric( task: dict[str, Any], output: dict[str, Any], @@ -236,11 +252,9 @@ def _score_c2_rubric( ) -> dict[str, float]: """Run C2 rubric scoring and return per-dimension scores. - Only the C2-owned dimensions are meaningful here: - ``insight_quality``, ``evidence_linkage``, and ``structure_usability``. - The ``calibration_limitation_handling`` and ``consistency`` values from - C2 are overridden by C3 and C4 respectively; ``grounding_accuracy`` is - overridden by C1. + Only the C2-owned dimensions are judged: ``insight_quality``, + ``evidence_linkage``, and ``structure_usability``. The remaining three + dimensions are owned by C1/C3/C4 and are never sent to the judge. Args: task: Task definition dict. @@ -249,15 +263,15 @@ def _score_c2_rubric( :class:`~benchmark.rubrics.rubric_scoring.JudgeClientProtocol`. Returns: - Dict mapping each of the six dimension names to a float in [0, 1]. - On error (e.g. malformed rubric), returns zero scores for all - dimensions. + Dict mapping each C2-owned dimension name to a float in [0, 1]. + On error (e.g. malformed rubric), returns zero scores for the + C2-owned dimensions. """ try: - result = score_rubric(task, output, judge_client) + result = score_rubric(task, output, judge_client, dimensions=_C2_OWNED_DIMENSIONS) return dict(result["dimension_scores"]) except (ValueError, KeyError): - return dict.fromkeys(RUBRIC_DIMENSIONS, 0.0) + return dict.fromkeys(_C2_OWNED_DIMENSIONS, 0.0) def _score_c3_calibration( @@ -407,7 +421,7 @@ def run_task( # --- C1: grounding accuracy via fact scoring --- c1_score = _score_c1_grounding(task, output) - # --- C2: rubric scoring (all six dimensions) --- + # --- C2: rubric scoring (judged dimensions only) --- c2_scores = _score_c2_rubric(task, output, effective_judge) # --- C3: claim validation → calibration_limitation_handling --- diff --git a/tests/unit/test_rubric_scoring.py b/tests/unit/test_rubric_scoring.py index 3d01918..78d0b79 100644 --- a/tests/unit/test_rubric_scoring.py +++ b/tests/unit/test_rubric_scoring.py @@ -252,6 +252,29 @@ def test_judge_called_once_per_dimension(self) -> None: score_rubric(_VALID_TASK, _VALID_OUTPUT, mock) assert mock.judge.call_count == len(RUBRIC_DIMENSIONS) + def test_dimensions_subset_judges_only_those(self) -> None: + """A dimensions subset must restrict judge calls and returned scores. + + The dispatcher passes only the C2-owned dimensions so judge calls for + C1/C3/C4-owned dimensions are never made (methodology finding M-2). + """ + subset = ("insight_quality", "evidence_linkage", "structure_usability") + mock = _make_mock_judge(0.5) + result = score_rubric(_VALID_TASK, _VALID_OUTPUT, mock, dimensions=subset) + + assert mock.judge.call_count == len(subset) + called_dims = {call.kwargs["dimension"] for call in mock.judge.call_args_list} + assert called_dims == set(subset) + assert set(result["dimension_scores"].keys()) == set(subset) + assert set(result["rubric_weights"].keys()) == set(subset) + + def test_dimensions_subset_unknown_dimension_raises(self) -> None: + """An unknown dimension in the subset must raise before judge calls.""" + mock = _make_mock_judge(0.5) + with pytest.raises(ValueError, match="Unknown rubric dimension"): + score_rubric(_VALID_TASK, _VALID_OUTPUT, mock, dimensions=("not_a_dim",)) + assert mock.judge.call_count == 0 + def test_judge_called_with_correct_dimension_names(self) -> None: """Each judge call must receive the correct dimension name as the first arg.""" mock = _make_mock_judge(0.5) diff --git a/tests/unit/test_runner_dispatcher.py b/tests/unit/test_runner_dispatcher.py index 030a132..15baf3a 100644 --- a/tests/unit/test_runner_dispatcher.py +++ b/tests/unit/test_runner_dispatcher.py @@ -283,6 +283,43 @@ def test_consistency_nonzero_for_multiple_runs(self) -> None: # Stub always emits the same first key_finding → consistency close to 1.0. assert result.scores.get("consistency", 0.0) > 0.0 + def test_c1_grounding_searches_limitations(self) -> None: + """A gold-fact value stated only in limitations must be credited (M-1).""" + from runner.dispatcher import _score_c1_grounding + + task = { + "task_id": "T-LIM", + "gold_facts": [ + { + "fact_id": "F1", + "claim": "The program enrolls 977 students.", + "source_files": ["students.csv"], + "numeric_value": 977, + "tolerance": 0, + } + ], + } + output = { + "structured_metrics": {}, + "key_findings": ["Enrollment is healthy this quarter."], + "limitations": ["Note that only 977 students are reflected in this snapshot."], + } + assert _score_c1_grounding(task, output) == pytest.approx(1.0) + + def test_c2_judge_not_called_for_non_c2_dimensions(self) -> None: + """The judge must only be called for C2-owned dimensions (M-2).""" + adapter = StubAdapter() + mock_judge = MagicMock() + mock_judge.judge.return_value = 0.5 + run_task(_VALID_TASK, adapter, runs=1, judge_client=mock_judge) + + called_dims = {call.kwargs["dimension"] for call in mock_judge.judge.call_args_list} + assert "grounding_accuracy" not in called_dims + assert "consistency" not in called_dims + # C3 may route calibration through the judge as a Stage-2 paraphrase + # fallback, but C2 itself must cover exactly the three owned dims. + assert {"insight_quality", "evidence_linkage", "structure_usability"} <= called_dims + # --------------------------------------------------------------------------- # C3 integration tests — calibration_limitation_handling driven by validate_claims From a09335503c4061731bb4b57a6c1a5f78017ff557 Mon Sep 17 00:00:00 2001 From: jonathanbbechtel Date: Wed, 10 Jun 2026 17:05:48 -0400 Subject: [PATCH 5/5] fix(scoring,adapter): close out high-severity methodology findings H-1..H-5 (#40) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(scoring): gate free-text numeric matches on claim context (H-1) Grounding accuracy credited a gold fact when ANY number anywhere in the output (up to 50 prose sentences, plus a /100 percent-normalized variant of every candidate) fell within tolerance. With the leniency floors, an unrelated "29.6%" anywhere in the response credited a gold count of 30 — rewarding number-dense outputs regardless of relevance on the highest-weighted dimension. Free-text numeric matches (key_findings / limitations) now require the containing sentence to share at least one content token with the gold claim (stopwords and numbers excluded, trailing plural-s normalized, hyphenated IDs like sch-001 preserved). An empty claim disables the gate. structured_metrics matching deliberately stays key-agnostic: structured values are deliberate model assertions, and key-name matching was previously found too brittle against real model outputs — the spam guard targets prose only. * fix(scoring): credit non-numeric facts via substring or token overlap (H-2) Non-numeric gold facts were located by substring containment but then re-scored with exact string equality — so a finding containing the claim plus any other words scored 0.0, making non-numeric facts near-universal misses unless the model echoed the claim verbatim. The module docstring promised token-overlap matching that was never implemented. The located match is now scored directly: stage 1 is substring containment (either direction, normalized); stage 2 credits paraphrases when >= 50% of the claim's content tokens appear in the entry (TEXT_FACT_OVERLAP_THRESHOLD, mirroring C3's claim-validation threshold). Match provenance is recorded in the method label (text_match[key_findings+token_overlap] etc.). * fix(scoring): exclude trivial consistency from single-run composites (H-3) With runs=1, all three C4 consistency sub-metrics trivially default to 1.0 — not measured consistency, just free credit (typically 10% of the composite) handed to every model equally, inflating absolute scores. When runs < 2 the dispatcher now stamps a consistency_trivial scorer flag and computes the composite with the consistency dimension excluded and the remaining weights renormalized, keeping single-run composites on the same [0, 1] scale. The per-dimension consistency score is still reported for schema completeness. Multi-run behavior is unchanged. * fix(adapter): raise max_tokens to 4096 + surface truncation via finish_reason (H-4) DEFAULT_MAX_TOKENS=1024 routinely truncated the multi-section prose analyses the prompt asks for. Limitations sections come last, so the cut silently deflated calibration_limitation_handling — and nothing recorded that truncation had happened. - DEFAULT_MAX_TOKENS 1024 -> 4096 (GRADE_OPENROUTER_MAX_TOKENS env var and constructor override unchanged) - The adapter now records the provider's finish_reason in runtime_metadata; output_schema.json gains the optional nullable field - The dispatcher stamps a "truncated_output" scorer flag when any run finishes with "length", making truncation visible in the scorecard * feat(adapter): candidate-side reasoning effort via @ model suffix (H-5) The launch plan runs GPT-5.5 at four reasoning-effort levels, but the adapter had no way to set an effort and no way to keep the four result sets distinct. - OpenRouterAdapter accepts reasoning_effort= and an inline @ model suffix (e.g. "openai/gpt-5.5@xhigh"); the suffix is stripped from the API slug but kept in the reported model_id so each effort level gets its own leaderboard row - Reasoning payload forwarded as {"reasoning": {"effort": ...}} - Reasoning runs default to a 16384-token budget (DEFAULT_REASONING_MAX_TOKENS) — reasoning tokens share the budget with the visible analysis, so the 4096 prose default would yield empty responses at high effort - @ syntax documented in --model (runner.cli) and --models (scripts.run_models) help; the sweep is now --models openai/gpt-5.5@xhigh,...,openai/gpt-5.5@low --------- Co-authored-by: jonathan bechtel --- benchmark/rubrics/fact_scoring.py | 188 +++++++++++++++++++++----- benchmark/schemas/output_schema.json | 4 + docs/methodology_review_findings.md | 113 +++++++++------- runner/adapters/openrouter_adapter.py | 55 ++++++-- runner/cli.py | 3 + runner/dispatcher.py | 32 ++++- scripts/run_models.py | 7 +- tests/unit/test_fact_scoring.py | 83 +++++++++++- tests/unit/test_openrouter_adapter.py | 74 ++++++++++ tests/unit/test_runner_dispatcher.py | 48 +++++++ tests/unit/test_runner_resilience.py | 19 +-- 11 files changed, 520 insertions(+), 106 deletions(-) diff --git a/benchmark/rubrics/fact_scoring.py b/benchmark/rubrics/fact_scoring.py index 0ca44f6..d016012 100644 --- a/benchmark/rubrics/fact_scoring.py +++ b/benchmark/rubrics/fact_scoring.py @@ -19,7 +19,11 @@ ``tolerance`` of ``gold_fact.numeric_value``. 2. If not found in ``structured_metrics``, parse numbers out of every string in ``key_findings`` and ``limitations`` using :func:`_extract_numbers`. Credit - if any parsed number is within tolerance. + if any parsed number is within tolerance — **but only when the containing + string shares at least one content token with the gold claim** (the + free-text context gate; see :func:`_shares_claim_context`). This prevents + numerically-close values in unrelated sentences from crediting the fact, + which would otherwise reward number-dense outputs regardless of relevance. False-positive guard for numeric matching ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,10 +93,12 @@ **Non-numeric gold facts** (``numeric_value`` is None): -Scans ``key_findings`` text (and ``limitations``) for the claim via normalized -substring matching: at least one token from the claim must appear in at least -one finding (token-overlap fallback) or the full claim/finding must be a -substring of the other after lowercasing and whitespace normalization. +Scans ``key_findings`` text (and ``limitations``) for the claim in two +stages: (1) normalized substring containment (either direction, after +lowercasing and whitespace stripping), then (2) token overlap — at least +:data:`TEXT_FACT_OVERLAP_THRESHOLD` (50%) of the claim's content tokens must +appear in the entry. A matching entry scores 1.0; paraphrases are credited +without requiring a verbatim echo of the claim. Public API ---------- @@ -531,6 +537,89 @@ def _extract_numbers(text: str) -> list[float]: # Fact-level dispatcher (value-based matching) # --------------------------------------------------------------------------- +#: Common function words excluded when extracting a claim's content tokens for +#: the free-text context gate. Deliberately small — only words that carry no +#: topical signal in benchmark claims. +_CONTEXT_STOPWORDS: frozenset[str] = frozenset( + { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "by", + "for", + "from", + "has", + "have", + "in", + "is", + "it", + "its", + "of", + "on", + "or", + "per", + "than", + "that", + "the", + "their", + "this", + "to", + "was", + "were", + "with", + } +) + +_TOKEN_PATTERN = re.compile(r"[a-z0-9][a-z0-9\-]*") + + +def _content_tokens(text: str) -> set[str]: + """Extract normalized content tokens from *text* for the context gate. + + Lowercases, splits on non-alphanumerics (keeping in-word hyphens, so IDs + like ``sch-001`` survive), drops stopwords, pure numbers, and 1-2 char + fragments, and normalizes a trailing plural ``s`` so ``student`` matches + ``students``. + + Args: + text: Arbitrary text (a gold claim or a model sentence). + + Returns: + A set of normalized content tokens (possibly empty). + """ + tokens: set[str] = set() + for tok in _TOKEN_PATTERN.findall(text.lower()): + if tok in _CONTEXT_STOPWORDS or len(tok) < 3: + continue + if tok.replace("-", "").replace(".", "").isdigit(): + continue + tokens.add(tok[:-1] if len(tok) > 3 and tok.endswith("s") else tok) + return tokens + + +def _shares_claim_context(claim_tokens: set[str], sentence: str) -> bool: + """Return True when *sentence* shares at least one content token with the claim. + + This is the free-text false-positive gate: a number found in prose is only + credited when its containing sentence is topically related to the gold + claim. When the claim yields no content tokens at all (pathological), the + gate is open — gating on nothing would reject everything. + + Args: + claim_tokens: Output of :func:`_content_tokens` for the gold claim. + sentence: The candidate finding/limitation string. + + Returns: + ``True`` if the sentence passes the context gate. + """ + if not claim_tokens: + return True + return bool(claim_tokens & _content_tokens(sentence)) + def _find_predicted_numeric( gold_value: float, @@ -538,6 +627,7 @@ def _find_predicted_numeric( structured_metrics: dict[str, Any], key_findings: list[str], limitations: list[str], + claim: str = "", ) -> tuple[float | None, str]: """Search for a numeric value matching *gold_value* within *tolerance*. @@ -547,6 +637,18 @@ def _find_predicted_numeric( 2. Numbers parsed from each string in ``key_findings``. 3. Numbers parsed from each string in ``limitations``. + Free-text context gate (false-positive guard) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Numbers parsed from free text (``key_findings`` / ``limitations``) are + only credited when the containing string shares at least one content + token with the gold *claim* (see :func:`_content_tokens`). Without the + gate, any numerically-close value anywhere in a long prose response — + up to 50 sentences — credits the fact, which rewards number-dense + outputs regardless of relevance. ``structured_metrics`` matching stays + fully key-agnostic: structured values are deliberate model assertions + (and key-name matching was previously found too brittle against real + model outputs), so the spam guard targets prose only. + For each candidate value ``N``, two comparisons are attempted: - **As-is**: ``score_numeric(N, gold_value, tolerance)``. @@ -566,6 +668,8 @@ def _find_predicted_numeric( structured_metrics: ``structured_metrics`` dict from the model output. key_findings: ``key_findings`` list from the model output. limitations: ``limitations`` list from the model output. + claim: The gold fact's claim text, used for the free-text context + gate. An empty claim disables the gate (matches old behavior). Returns: ``(matched_value, source_label)`` where *source_label* is one of @@ -576,6 +680,7 @@ def _find_predicted_numeric( """ # Whether to attempt the /100 normalization (only safe for fraction/rate gold values) try_pct_norm: bool = 0 < gold_value <= 1 + claim_tokens: set[str] = _content_tokens(claim) def _matches_candidate(candidate: float) -> str: """Return '' if no match, 'as_is' or 'pct_norm' for the matching form.""" @@ -594,27 +699,27 @@ def _matches_candidate(candidate: float) -> str: if match_kind == "pct_norm": return (float(val), "structured_metrics+percent_normalized") - # 2. Parse numbers from key_findings free text - for finding in key_findings: - for num in _extract_numbers(finding): - match_kind = _matches_candidate(num) - if match_kind == "as_is": - return (num, "key_findings") - if match_kind == "pct_norm": - return (num, "key_findings+percent_normalized") - - # 3. Parse numbers from limitations free text - for lim in limitations: - for num in _extract_numbers(lim): - match_kind = _matches_candidate(num) - if match_kind == "as_is": - return (num, "limitations") - if match_kind == "pct_norm": - return (num, "limitations+percent_normalized") + # 2./3. Parse numbers from free text, gated on claim-context overlap. + for text_list, label in ((key_findings, "key_findings"), (limitations, "limitations")): + for entry in text_list: + if not _shares_claim_context(claim_tokens, entry): + continue + for num in _extract_numbers(entry): + match_kind = _matches_candidate(num) + if match_kind == "as_is": + return (num, label) + if match_kind == "pct_norm": + return (num, f"{label}+percent_normalized") return (None, "not_found") +#: Minimum share of the claim's content tokens that must appear in a finding +#: for a non-numeric fact to be credited via token overlap. Mirrors C3's +#: ``TOKEN_OVERLAP_THRESHOLD`` in :mod:`benchmark.rubrics.claim_validation`. +TEXT_FACT_OVERLAP_THRESHOLD: float = 0.5 + + def _find_predicted_text( claim: str, key_findings: list[str], @@ -622,9 +727,14 @@ def _find_predicted_text( ) -> tuple[str | None, str]: """Search for a non-numeric claim in ``key_findings`` and ``limitations``. - Matching uses normalized substring containment (case-insensitive, leading/ - trailing whitespace stripped). The claim is credited when either the claim - is a substring of a finding or the finding is a substring of the claim. + Two matching stages, applied per entry (substring takes precedence): + + 1. **Substring containment** (case-insensitive, whitespace-stripped): + the claim is contained in the entry or vice-versa. + 2. **Token overlap**: at least :data:`TEXT_FACT_OVERLAP_THRESHOLD` of the + claim's content tokens (see :func:`_content_tokens`) appear in the + entry. This credits paraphrases — the dominant way real models state + non-numeric facts — without requiring a verbatim echo of the claim. Args: claim: The gold fact claim text. @@ -632,15 +742,22 @@ def _find_predicted_text( limitations: ``limitations`` list from the model output. Returns: - ``(matched_text, source_label)`` or ``(None, 'not_found')``. + ``(matched_text, source_label)`` or ``(None, 'not_found')`` where + *source_label* is ``'key_findings'``, ``'limitations'``, or their + ``'+token_overlap'``-suffixed variants. """ claim_lower = claim.strip().lower() + claim_tokens = _content_tokens(claim) for text_list, label in [(key_findings, "key_findings"), (limitations, "limitations")]: for entry in text_list: entry_lower = entry.strip().lower() if claim_lower in entry_lower or entry_lower in claim_lower: return (entry, label) + if claim_tokens: + overlap = len(claim_tokens & _content_tokens(entry)) / len(claim_tokens) + if overlap >= TEXT_FACT_OVERLAP_THRESHOLD: + return (entry, f"{label}+token_overlap") return (None, "not_found") @@ -654,8 +771,8 @@ def score_fact( """Score one gold fact against a model output. Dispatches to :func:`score_numeric` when ``gold_fact.numeric_value`` is - not ``None``, and to :func:`score_exact_match` for non-numeric (string) - facts. + not ``None``, and to :func:`_find_predicted_text` (substring or token + overlap) for non-numeric (string) facts. For **numeric** facts the effective tolerance is: @@ -670,7 +787,8 @@ def score_fact( For **non-numeric** facts the predicted value is taken from the first ``key_findings`` (or ``limitations``) entry that contains the claim as a - substring (or vice-versa); if neither yields a value the fact scores ``0.0``. + substring (or vice-versa) or shares >= 50% of the claim's content tokens; + if neither yields a value the fact scores ``0.0``. Args: gold_fact: The gold fact to evaluate. @@ -720,6 +838,7 @@ def score_fact( structured_metrics, key_findings, lims, + claim=gold_fact.claim, ) if matched_val is None: @@ -753,13 +872,16 @@ def score_fact( method="not_found", ) - s = score_exact_match(matched_text, gold_fact.claim) + # The find already verified the match (substring or token overlap) — + # score it directly. Re-checking with exact string equality here + # (the old behavior) made non-numeric facts near-impossible to credit + # unless the model echoed the claim verbatim. return FactDetail( fact_id=gold_fact.fact_id, - score=s, - matched=s == 1.0, + score=1.0, + matched=True, predicted_value=matched_text, - method=f"exact_match[{source}]", + method=f"text_match[{source}]", ) diff --git a/benchmark/schemas/output_schema.json b/benchmark/schemas/output_schema.json index 6d230b8..dde452a 100644 --- a/benchmark/schemas/output_schema.json +++ b/benchmark/schemas/output_schema.json @@ -150,6 +150,10 @@ "pack_id": { "type": ["string", "null"], "description": "Fixture pack identifier the task was run against, e.g. 'pack_operations', 'pack_outcomes', 'pack_equity_research'." + }, + "finish_reason": { + "type": ["string", "null"], + "description": "Provider-reported finish reason for the completion, e.g. 'stop' or 'length'. 'length' means the response hit max_tokens and was truncated. Null/absent when the provider does not report one." } } } diff --git a/docs/methodology_review_findings.md b/docs/methodology_review_findings.md index c69f951..7ee2382 100644 --- a/docs/methodology_review_findings.md +++ b/docs/methodology_review_findings.md @@ -85,58 +85,71 @@ reasoning tokens share the budget with the one-float answer). ## High (affects score validity — fix before the real run) -### H-1. Grounding accuracy is credit-only and key-agnostic → rewards number spam — `OPEN` -`benchmark/rubrics/fact_scoring.py`: a gold fact is credited if **any** +### H-1. Grounding accuracy is credit-only and key-agnostic → rewards number spam — `FIXED` +`benchmark/rubrics/fact_scoring.py`: a gold fact was credited if **any** number anywhere in the output (up to 50 prose sentences, plus a `/100` -percent-normalized variant of every candidate) falls within tolerance. -Leniency floors widen the window further (gold count 30 with authored -tolerance 0 gets ±0.6, so an unrelated "29.6%" anywhere credits it). There is -no penalty for wrong/hallucinated numbers. Net effect: verbose, number-dense -outputs systematically outscore concise ones on the highest-weighted -dimension (0.35–0.40). - -**Suggested fix:** require the matching number to appear in a sentence with -token overlap against the gold claim (or within N tokens of a claim keyword), -and/or add a precision-style penalty for confidently asserted numbers that -match no gold fact. At minimum, log per-fact `method` labels into the result -for auditability. - -### H-2. Non-numeric gold facts effectively require verbatim echo — `OPEN` -`fact_scoring.py:score_fact` finds a candidate finding by substring -containment but then scores it with **exact** string equality -(`score_exact_match`), so a finding that contains the claim plus any other -words scores 0. The module docstring promises token-overlap matching that is -not implemented. Non-numeric facts are near-universal misses for all models. - -**Suggested fix:** score the substring/token-overlap match directly (like -C3's `TOKEN_OVERLAP_THRESHOLD` approach) instead of exact equality. - -### H-3. `runs=1` gives every model a free 10% (consistency) — `OPEN` -All three consistency sub-metrics default to 1.0 with a single run. The -leaderboard run must use `--runs ≥ 3` (default 5 is good). Suggested fix: -stamp a `consistency_trivial` scorer flag when `runs < 2`, or exclude the -dimension from the composite in that case. - -### H-4. `max_tokens=1024` truncates analyses; reasoning models break — `OPEN` -`OpenRouterAdapter.DEFAULT_MAX_TOKENS = 1024` is tight for a multi-section -prose analysis; limitations sections come last and get cut first (deflating -`calibration_limitation_handling`). `finish_reason` is not checked, so -truncation is invisible. Reasoning models (GPT-5.5 at high effort) can burn -the entire budget on reasoning tokens and return an empty visible response. - -**Suggested fix:** raise the default to ≥ 4096, check -`choices[0].finish_reason == "length"` and stamp a `truncated` flag, and add -a `reasoning` parameter passthrough (see H-5). - -### H-5. No reasoning-effort support — required for the GPT-5.5 sweep — `OPEN` (partial) +percent-normalized variant of every candidate) fell within tolerance. +Leniency floors widened the window further (gold count 30 with authored +tolerance 0 gets ±0.6, so an unrelated "29.6%" anywhere credited it). + +**Fix applied:** free-text numeric matches (`key_findings` / `limitations`) +now pass a **claim-context gate** — the containing sentence must share at +least one content token with the gold claim (stopwords/numbers excluded, +plural-`s` normalized; see `_shares_claim_context`). The "29.6% of survey +responses arrived late" example no longer credits a gold count of 30. +`structured_metrics` matching deliberately stays key-agnostic: structured +values are deliberate model assertions, and key-name matching was previously +found too brittle against real model outputs. Covered by the new +context-gate tests in `test_fact_scoring.py`. + +### H-2. Non-numeric gold facts effectively require verbatim echo — `FIXED` +`fact_scoring.py:score_fact` found a candidate finding by substring +containment but then scored it with **exact** string equality, so a finding +containing the claim plus any other words scored 0 — non-numeric facts were +near-universal misses. + +**Fix applied:** the match is now scored directly. Stage 1 is substring +containment (either direction); stage 2 credits paraphrases when >= 50% of +the claim's content tokens appear in the entry +(`TEXT_FACT_OVERLAP_THRESHOLD`, mirroring C3). Match provenance is recorded +in the `method` label (`text_match[key_findings+token_overlap]` etc.). The +doc/code mismatch (docstring promised token overlap) is resolved. + +### H-3. `runs=1` gives every model a free 10% (consistency) — `FIXED` +All three consistency sub-metrics default to 1.0 with a single run. + +**Fix applied:** when `runs < 2` the dispatcher stamps a +`consistency_trivial` scorer flag and computes the composite with the +consistency dimension excluded and the remaining weights renormalized — so +single-run composites stay on the same [0, 1] scale without free credit. +The per-dimension `consistency` score is still reported (1.0) for schema +completeness. Multi-run behavior is unchanged. The leaderboard run should +still use `--runs 5`. + +### H-4. `max_tokens=1024` truncates analyses; reasoning models break — `FIXED` +`OpenRouterAdapter.DEFAULT_MAX_TOKENS = 1024` was tight for a multi-section +prose analysis; limitations sections come last and got cut first (deflating +`calibration_limitation_handling`), and truncation was invisible. + +**Fix applied:** default raised to 4096 (env/constructor overrides +unchanged); the adapter records the provider's `finish_reason` in +`runtime_metadata` (new optional schema field), and the dispatcher stamps a +`truncated_output` scorer flag whenever any run finishes with `"length"` — +so truncation shows up in the scorecard instead of silently deflating +scores. Reasoning-model budgets are handled in H-5. + +### H-5. No reasoning-effort support — required for the GPT-5.5 sweep — `FIXED` The launch plan includes GPT-5.5 at xhigh/high/medium/low effort. -**Done (2026-06-10):** `post_chat_completion` now accepts a -`reasoning_effort` parameter (used by the GPT-5.5 judge). -**Still open:** `OpenRouterAdapter` (the candidate-side path) does not yet -accept/forward a reasoning effort, and results need distinct `model_id` -labels per effort level (e.g. `openai/gpt-5.5@xhigh`) so leaderboard rows -don't collide. Wire the adapter through the same parameter and add a -`--reasoning-effort` CLI flag (or per-model syntax in `--models`). + +**Fix applied:** `OpenRouterAdapter` accepts a `reasoning_effort` parameter +and an inline `@` model suffix (e.g. `openai/gpt-5.5@xhigh`, +documented in both CLIs' help). The suffix is stripped from the API slug but +kept in the reported `model_id`, so the same slug at different efforts gets +distinct leaderboard rows. Reasoning runs default to a 16384-token budget +(`DEFAULT_REASONING_MAX_TOKENS`) since reasoning tokens share the budget +with the visible analysis. The judge-side passthrough landed earlier +(`post_chat_completion`). The GPT-5.5 sweep is now: +`--models openai/gpt-5.5@xhigh,openai/gpt-5.5@high,openai/gpt-5.5@medium,openai/gpt-5.5@low`. --- diff --git a/runner/adapters/openrouter_adapter.py b/runner/adapters/openrouter_adapter.py index 9e0cac9..2487a08 100644 --- a/runner/adapters/openrouter_adapter.py +++ b/runner/adapters/openrouter_adapter.py @@ -407,9 +407,19 @@ class OpenRouterAdapter: #: CLI id — ``--adapter openrouter``. name: str = "openrouter" - #: Default maximum tokens — lower than the original 2048 to reduce cost. - #: Override via ``GRADE_OPENROUTER_MAX_TOKENS`` env var or the constructor arg. - DEFAULT_MAX_TOKENS: int = 1024 + #: Default maximum tokens. The prompt asks for a multi-section prose + #: analysis of full CSV fixtures; 1024 (the old default) routinely + #: truncated responses — and limitations sections come last, so the cut + #: silently deflated calibration scores. Override via the + #: ``GRADE_OPENROUTER_MAX_TOKENS`` env var or the constructor arg. + DEFAULT_MAX_TOKENS: int = 4096 + + #: Default maximum tokens when a reasoning effort is set. Reasoning + #: tokens and the visible analysis share this budget, so reasoning models + #: need substantially more headroom than :attr:`DEFAULT_MAX_TOKENS` — at + #: high effort the model can otherwise burn the whole budget thinking and + #: return an empty visible response. + DEFAULT_REASONING_MAX_TOKENS: int = 16384 def __init__( self, @@ -417,11 +427,16 @@ def __init__( api_key: str | None = None, temperature: float = 1.0, max_tokens: int | None = None, + reasoning_effort: str | None = None, ) -> None: """Initialise the adapter. Args: - model: OpenRouter model slug or shorthand key. + model: OpenRouter model slug or shorthand key. May carry an + ``@`` suffix (e.g. ``"openai/gpt-5.5@xhigh"``) to set + the reasoning effort inline — the suffix is stripped from the + API slug but kept in the reported ``model_id`` so leaderboard + rows for different effort levels don't collide. api_key: API key; falls back to ``OPENROUTER_API_KEY`` env var. temperature: Sampling temperature for the model call. Defaults to ``1.0`` so that repeated benchmark runs vary, making the @@ -429,17 +444,35 @@ def __init__( max_tokens: Maximum tokens in the model response. When ``None`` (the default), the value is resolved from the ``GRADE_OPENROUTER_MAX_TOKENS`` environment variable if set, - otherwise :attr:`DEFAULT_MAX_TOKENS` (1024) is used. + otherwise :attr:`DEFAULT_MAX_TOKENS` (4096) — or + :attr:`DEFAULT_REASONING_MAX_TOKENS` (16384) when a reasoning + effort is in play, since reasoning tokens share the budget. + reasoning_effort: Optional reasoning effort level (``"low"``, + ``"medium"``, ``"high"``, ``"xhigh"``) forwarded to + OpenRouter as ``{"reasoning": {"effort": ...}}``. Takes + precedence over an ``@`` suffix in *model*. """ self._model_input = model - self._model = _resolve_model(model) + base_model, _, suffix_effort = model.partition("@") + self._reasoning_effort = reasoning_effort or (suffix_effort or None) + self._model = _resolve_model(base_model) + #: model_id reported in outputs — includes the effort label so results + #: for the same slug at different efforts stay distinct. + self._model_label = ( + f"{self._model}@{self._reasoning_effort}" if self._reasoning_effort else self._model + ) self._api_key = api_key # resolved lazily in run() to support env var self._temperature = temperature if max_tokens is not None: self._max_tokens = max_tokens else: env_val = os.environ.get("GRADE_OPENROUTER_MAX_TOKENS") - self._max_tokens = int(env_val) if env_val else self.DEFAULT_MAX_TOKENS + if env_val: + self._max_tokens = int(env_val) + elif self._reasoning_effort is not None: + self._max_tokens = self.DEFAULT_REASONING_MAX_TOKENS + else: + self._max_tokens = self.DEFAULT_MAX_TOKENS def _get_api_key(self) -> str: """Resolve and return the API key. @@ -503,6 +536,8 @@ def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: "temperature": self._temperature, "max_tokens": self._max_tokens, } + if self._reasoning_effort is not None: + payload["reasoning"] = {"effort": self._reasoning_effort} t_start = time.monotonic() response = httpx.post( @@ -524,6 +559,7 @@ def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: body: dict[str, Any] = response.json() choice = body["choices"][0] raw_text: str = choice["message"]["content"] or "" + finish_reason: str | None = choice.get("finish_reason") usage: dict[str, Any] = body.get("usage", {}) prompt_tokens: int | None = usage.get("prompt_tokens") @@ -552,11 +588,14 @@ def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: "model_temperature": self._temperature, "provider": provider, "pack_id": None, + # "length" means the response hit max_tokens and was truncated — + # the dispatcher stamps a 'truncated_output' scorer flag from this. + "finish_reason": finish_reason, } return { "task_id": task["task_id"], - "model_id": self._model, + "model_id": self._model_label, "run_index": run_index, "structured_metrics": parsed["structured_metrics"], "key_findings": parsed["key_findings"], diff --git a/runner/cli.py b/runner/cli.py index a92699e..6b0bedf 100644 --- a/runner/cli.py +++ b/runner/cli.py @@ -198,6 +198,9 @@ def build_parser() -> argparse.ArgumentParser: help=( "Model slug or shorthand forwarded to the adapter " "(e.g. 'anthropic/claude-sonnet-4.6', 'claude-sonnet-4-6'). " + "Append @ to set a reasoning effort for reasoning models " + "(e.g. 'openai/gpt-5.5@xhigh') — the effort is kept in the " + "reported model_id so leaderboard rows stay distinct. " "Required for the openrouter adapter; ignored by the stub." ), ) diff --git a/runner/dispatcher.py b/runner/dispatcher.py index ebdbe14..8ac945e 100644 --- a/runner/dispatcher.py +++ b/runner/dispatcher.py @@ -298,20 +298,32 @@ def _score_c3_calibration( def _compute_composite( scores: DimensionScores, rubric: dict[str, Any], + exclude: tuple[str, ...] = (), ) -> float: """Compute the weighted composite score from per-dimension scores. Args: scores: Per-dimension scores in [0, 1]. rubric: Task rubric dict with per-dimension ``weight`` values. + exclude: Dimensions to drop from the composite. The remaining + weights are renormalized to sum to 1, so the composite stays on + the same [0, 1] scale. Used for single-run executions where + ``consistency`` is trivially 1.0 and would otherwise be free + credit. Returns: Weighted sum in [0, 1]. """ total = 0.0 + included_weight = 0.0 for dim in RUBRIC_DIMENSIONS: + if dim in exclude: + continue weight = float(rubric.get(dim, {}).get("weight", 0.0)) + included_weight += weight total += weight * scores.get(dim, 0.0) + if exclude and included_weight > 0: + return total / included_weight return total @@ -418,6 +430,14 @@ def run_task( validate_output(output) outputs.append(output) + # Surface truncation: a finish_reason of "length" means the response + # hit max_tokens mid-analysis. Limitations sections come last in + # prose responses, so truncation silently deflates calibration scores + # — make it visible in the scorecard instead. + if output.get("runtime_metadata", {}).get("finish_reason") == "length": + if "truncated_output" not in all_flags: + all_flags.append("truncated_output") + # --- C1: grounding accuracy via fact scoring --- c1_score = _score_c1_grounding(task, output) @@ -450,8 +470,16 @@ def run_task( values = [s.get(dim, 0.0) for s in per_run_scores] avg_scores[dim] = sum(values) / len(values) if values else 0.0 - # Compute composite. - composite = _compute_composite(avg_scores, rubric) + # Compute composite. With a single run, all three C4 sub-metrics + # trivially default to 1.0 — that's not measured consistency, it's free + # credit (typically 10% of the composite). Exclude the dimension and + # renormalize the remaining weights so single-run composites stay + # comparable, and flag the run so downstream consumers can tell. + if runs < 2: + all_flags.append("consistency_trivial") + composite = _compute_composite(avg_scores, rubric, exclude=("consistency",)) + else: + composite = _compute_composite(avg_scores, rubric) return TaskRunResult( task_id=task_id, diff --git a/scripts/run_models.py b/scripts/run_models.py index afaa0a2..9a864c6 100644 --- a/scripts/run_models.py +++ b/scripts/run_models.py @@ -190,8 +190,11 @@ def build_parser() -> argparse.ArgumentParser: required=True, metavar="M1,M2,...", help=( - "Comma-separated list of model IDs to evaluate, " - "e.g. 'anthropic/claude-haiku-3,anthropic/claude-sonnet-4-5'." + "Comma-separated list of model IDs to evaluate, e.g. " + "'anthropic/claude-sonnet-4.6,openai/gpt-5.5@xhigh'. Append " + "@ to run a reasoning model at a specific effort level; " + "the same slug may appear at several efforts and each gets its " + "own leaderboard row." ), ) parser.add_argument( diff --git a/tests/unit/test_fact_scoring.py b/tests/unit/test_fact_scoring.py index a8ac9d2..8db7c0a 100644 --- a/tests/unit/test_fact_scoring.py +++ b/tests/unit/test_fact_scoring.py @@ -467,9 +467,56 @@ def test_numeric_relative_tolerance_credited(self) -> None: assert _sn(106.0, 100.0, "5%") == 0.0 def test_numeric_value_in_limitations_text(self) -> None: - """Gold numeric fact should be credited when value appears only in limitations text.""" + """Gold numeric fact should be credited when value appears only in limitations text. + + The limitation sentence must share claim context ("students") to pass + the free-text gate. + """ fact = self._make_fact(numeric_value=135.0, tolerance=0.0) - detail = score_fact(fact, {}, [], limitations=["Note: 135 records were processed."]) + detail = score_fact(fact, {}, [], limitations=["Note: only 135 students were counted."]) + assert detail.score == 1.0 + assert detail.matched is True + + # --- Free-text context gate (H-1 false-positive guard) --- + + def test_unrelated_sentence_number_not_credited(self) -> None: + """A matching number in a topically unrelated sentence must NOT credit the fact.""" + fact = self._make_fact( + claim="School SCH-003 has 30 enrolled students.", + numeric_value=30.0, + tolerance=0.0, + ) + # 29.6% would match gold 30 via the 2% count floor (±0.6), but the + # sentence shares no content token with the claim. + detail = score_fact(fact, {}, ["29.6% of survey responses arrived late."]) + assert detail.score == 0.0 + assert detail.matched is False + + def test_related_sentence_number_credited(self) -> None: + """The same number IS credited when its sentence shares claim context.""" + fact = self._make_fact( + claim="School SCH-003 has 30 enrolled students.", + numeric_value=30.0, + tolerance=0.0, + ) + detail = score_fact(fact, {}, ["SCH-003 enrolls 30 students."]) + assert detail.score == 1.0 + assert detail.matched is True + + def test_plural_singular_token_normalization(self) -> None: + """The gate must treat 'student' and 'students' as the same token.""" + fact = self._make_fact( + claim="The program enrolls 135 students.", numeric_value=135.0, tolerance=0.0 + ) + detail = score_fact(fact, {}, ["Each student is counted once: 135 in total."]) + assert detail.score == 1.0 + + def test_structured_metrics_stay_key_agnostic(self) -> None: + """The context gate applies to free text only — structured_metrics keys are ignored.""" + fact = self._make_fact( + claim="The program enrolls 135 students.", numeric_value=135.0, tolerance=0.0 + ) + detail = score_fact(fact, {"completely_unrelated_key": 135}, []) assert detail.score == 1.0 assert detail.matched is True @@ -490,6 +537,38 @@ def test_non_numeric_fact_not_found_scores_zero(self) -> None: assert detail.score == 0.0 assert detail.method == "not_found" + def test_non_numeric_paraphrase_credited_via_token_overlap(self) -> None: + """A paraphrase sharing >= 50% of the claim's content tokens must score 1.0 (H-2). + + Under the old behavior the finding was located by substring/overlap but + then re-scored with exact string equality — so anything short of a + verbatim echo scored 0.0. + """ + claim = "Group counts describe the current active roster." + fact = GoldFact("F7", claim, ["groups.csv"], None, None) + # Shares content tokens: group, count, describe, active, roster (not verbatim). + finding = "These group counts describe only the active roster as of today." + detail = score_fact(fact, {}, [finding]) + assert detail.score == 1.0 + assert detail.matched is True + assert "token_overlap" in detail.method or "text_match" in detail.method + + def test_non_numeric_superstring_finding_credited(self) -> None: + """A finding that CONTAINS the claim plus extra words must score 1.0 (H-2).""" + claim = "SCH-001 has 6 tutoring groups" + fact = GoldFact("F6", claim, ["groups.csv"], None, None) + detail = score_fact(fact, {}, [f"As shown in groups.csv, {claim}, the most of any site."]) + assert detail.score == 1.0 + assert detail.matched is True + + def test_non_numeric_low_overlap_not_credited(self) -> None: + """A finding sharing < 50% of the claim's content tokens must NOT be credited.""" + claim = "Group counts describe the current active roster snapshot for tutors." + fact = GoldFact("F8", claim, ["groups.csv"], None, None) + detail = score_fact(fact, {}, ["The roster was updated."]) # 1 of ~7 content tokens + assert detail.score == 0.0 + assert detail.matched is False + # --- Missing value --- def test_missing_value_scores_zero(self) -> None: diff --git a/tests/unit/test_openrouter_adapter.py b/tests/unit/test_openrouter_adapter.py index 5bf8d6a..1eee7d8 100644 --- a/tests/unit/test_openrouter_adapter.py +++ b/tests/unit/test_openrouter_adapter.py @@ -409,6 +409,80 @@ def test_latency_non_negative(self) -> None: output = self._run_with_mock() assert output["runtime_metadata"]["latency_ms"] >= 0.0 + def test_finish_reason_captured_and_schema_valid(self) -> None: + """finish_reason must be recorded in runtime_metadata and pass schema validation (H-4).""" + from benchmark.schemas import validate_output + + output = self._run_with_mock() + assert output["runtime_metadata"]["finish_reason"] == "stop" + validate_output(output) + + def test_truncated_finish_reason_recorded(self) -> None: + """A 'length' finish_reason (truncated response) must be recorded verbatim.""" + truncated_body = { + **_MOCK_OR_RESPONSE, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "## Key Findings\n- Cut off"}, + "finish_reason": "length", + } + ], + } + output = self._run_with_mock(mock_body=truncated_body) + assert output["runtime_metadata"]["finish_reason"] == "length" + + def test_default_max_tokens_fits_prose_analysis(self) -> None: + """The default max_tokens must be >= 4096 (1024 truncated real analyses, H-4).""" + assert OpenRouterAdapter.DEFAULT_MAX_TOKENS >= 4096 + with _MockHttpx(_MOCK_OR_RESPONSE) as mock_post: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + adapter.run(_VALID_TASK, run_index=0) + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert payload["max_tokens"] >= 4096 + + # --- Reasoning effort (H-5) --- + + def test_effort_suffix_parsed_into_payload_and_label(self) -> None: + """'slug@effort' must send the bare slug + reasoning field, with @effort in model_id.""" + with _MockHttpx(_MOCK_OR_RESPONSE) as mock_post: + adapter = OpenRouterAdapter(model="openai/gpt-5.5@xhigh", api_key="sk-or-test") + output = adapter.run(_VALID_TASK, run_index=0) + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert payload["model"] == "openai/gpt-5.5" # bare slug to the API + assert payload["reasoning"] == {"effort": "xhigh"} + assert output["model_id"] == "openai/gpt-5.5@xhigh" # labeled result + + def test_explicit_reasoning_effort_param(self) -> None: + """The reasoning_effort constructor arg must work without the @ suffix.""" + with _MockHttpx(_MOCK_OR_RESPONSE) as mock_post: + adapter = OpenRouterAdapter( + model="openai/gpt-5.5", api_key="sk-or-test", reasoning_effort="low" + ) + output = adapter.run(_VALID_TASK, run_index=0) + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert payload["reasoning"] == {"effort": "low"} + assert output["model_id"] == "openai/gpt-5.5@low" + + def test_reasoning_effort_raises_default_token_budget(self) -> None: + """Reasoning runs must default to the larger token budget (reasoning shares it).""" + adapter = OpenRouterAdapter(model="openai/gpt-5.5@xhigh", api_key="sk-or-test") + assert adapter._max_tokens == OpenRouterAdapter.DEFAULT_REASONING_MAX_TOKENS + # Explicit max_tokens still wins. + adapter2 = OpenRouterAdapter( + model="openai/gpt-5.5@xhigh", api_key="sk-or-test", max_tokens=2048 + ) + assert adapter2._max_tokens == 2048 + + def test_no_effort_omits_reasoning_field(self) -> None: + """Without an effort, the payload must not carry a reasoning field.""" + with _MockHttpx(_MOCK_OR_RESPONSE) as mock_post: + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + output = adapter.run(_VALID_TASK, run_index=0) + payload = mock_post.call_args.kwargs.get("json") or mock_post.call_args[1].get("json") + assert "reasoning" not in payload + assert "@" not in output["model_id"] + def test_token_counts_from_usage(self) -> None: """prompt_tokens and completion_tokens must be populated from usage.""" output = self._run_with_mock() diff --git a/tests/unit/test_runner_dispatcher.py b/tests/unit/test_runner_dispatcher.py index 15baf3a..3465002 100644 --- a/tests/unit/test_runner_dispatcher.py +++ b/tests/unit/test_runner_dispatcher.py @@ -276,6 +276,32 @@ def test_consistency_is_one_for_single_run(self) -> None: result = run_task(_VALID_TASK, adapter, runs=1) assert result.scores.get("consistency") == pytest.approx(1.0) + def test_single_run_flags_trivial_consistency_and_excludes_it(self) -> None: + """runs=1 must stamp consistency_trivial and drop the dimension from the composite (H-3). + + The composite must equal the renormalized weighted sum of the other + five dimensions — a trivial consistency=1.0 must contribute nothing. + """ + adapter = StubAdapter() + result = run_task(_VALID_TASK, adapter, runs=1) + assert "consistency_trivial" in result.scorer_flags + + rubric = _VALID_TASK["rubric"] + included = [d for d in result.scores if d != "consistency"] + weight_sum = sum(rubric[d]["weight"] for d in included) + expected = sum(rubric[d]["weight"] * result.scores[d] for d in included) / weight_sum + assert result.composite == pytest.approx(expected) + + def test_multi_run_composite_includes_consistency(self) -> None: + """runs>=2 must keep consistency in the composite and not flag it.""" + adapter = StubAdapter() + result = run_task(_VALID_TASK, adapter, runs=2) + assert "consistency_trivial" not in result.scorer_flags + + rubric = _VALID_TASK["rubric"] + expected = sum(rubric[d]["weight"] * result.scores[d] for d in result.scores) + assert result.composite == pytest.approx(expected) + def test_consistency_nonzero_for_multiple_runs(self) -> None: """Consistency with identical stub outputs should be > 0.""" adapter = StubAdapter() @@ -283,6 +309,28 @@ def test_consistency_nonzero_for_multiple_runs(self) -> None: # Stub always emits the same first key_finding → consistency close to 1.0. assert result.scores.get("consistency", 0.0) > 0.0 + def test_truncated_output_flagged(self) -> None: + """A run whose finish_reason is 'length' must stamp the truncated_output flag (H-4).""" + + class _TruncatingAdapter: + name = "stub" + + def __init__(self) -> None: + self._inner = StubAdapter() + + def run(self, task: dict[str, Any], run_index: int = 0) -> dict[str, Any]: + output = self._inner.run(task, run_index=run_index) + output["runtime_metadata"]["finish_reason"] = "length" + return output + + result = run_task(_VALID_TASK, _TruncatingAdapter(), runs=2) + assert "truncated_output" in result.scorer_flags + + def test_non_truncated_output_not_flagged(self) -> None: + """Normal stub runs must not carry the truncated_output flag.""" + result = run_task(_VALID_TASK, StubAdapter(), runs=1) + assert "truncated_output" not in result.scorer_flags + def test_c1_grounding_searches_limitations(self) -> None: """A gold-fact value stated only in limitations must be credited (M-1).""" from runner.dispatcher import _score_c1_grounding diff --git a/tests/unit/test_runner_resilience.py b/tests/unit/test_runner_resilience.py index 2274760..6d90171 100644 --- a/tests/unit/test_runner_resilience.py +++ b/tests/unit/test_runner_resilience.py @@ -4,8 +4,9 @@ - Partial run: adapter raises on one task → result.json contains only the successful tasks, failures.json lists the failed one, runner returns 0. - All-failure run: all tasks fail → runner returns 1, no result.json written. -- OpenRouterAdapter default max_tokens is 1024; constructor override and - GRADE_OPENROUTER_MAX_TOKENS env-var override both work. +- OpenRouterAdapter default max_tokens is 4096 (raised from 1024, which + truncated prose analyses — methodology finding H-4); constructor override + and GRADE_OPENROUTER_MAX_TOKENS env-var override both work. """ from __future__ import annotations @@ -337,16 +338,16 @@ def test_no_failures_json_when_all_succeed( class TestOpenRouterMaxTokens: - """OpenRouterAdapter default max_tokens is 1024 and can be overridden.""" + """OpenRouterAdapter default max_tokens is 4096 and can be overridden.""" - def test_default_max_tokens_is_1024(self) -> None: - """Default max_tokens must be 1024.""" - adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4-5", api_key="sk-or-test") - assert adapter._max_tokens == 1024 + def test_default_max_tokens_is_4096(self) -> None: + """Default max_tokens must be 4096 (1024 truncated prose analyses, H-4).""" + adapter = OpenRouterAdapter(model="anthropic/claude-sonnet-4.6", api_key="sk-or-test") + assert adapter._max_tokens == 4096 def test_default_max_tokens_class_constant(self) -> None: - """DEFAULT_MAX_TOKENS class attribute must be 1024.""" - assert OpenRouterAdapter.DEFAULT_MAX_TOKENS == 1024 + """DEFAULT_MAX_TOKENS class attribute must be 4096.""" + assert OpenRouterAdapter.DEFAULT_MAX_TOKENS == 4096 def test_constructor_override(self) -> None: """Explicit max_tokens constructor arg must override the default."""