From f6957cedfbafa31a258b5399d8269100109a2521 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sun, 2 Aug 2026 08:07:46 +0100 Subject: [PATCH 1/2] feat(gitchameleon): ready the funded evidence run Pin the Voyage baseline and upstream scorer, add immutable dataset fetches, safe resume semantics, provenance hashes, completeness gates, and the accepted execution decision. Authored under my direction with Codex. --- gitchameleon/README.md | 46 +- gitchameleon/answer.py | 67 ++- gitchameleon/arms.py | 125 +++++- .../decisions/GCB-ADR-0001-benchmark-scope.md | 10 +- .../GCB-ADR-0003-funded-run-execution.md | 63 +++ gitchameleon/fetch_dataset.py | 65 +-- gitchameleon/run-config.json | 46 ++ gitchameleon/run.py | 392 ++++++++++++++---- tests/test_gitchameleon_scaffold.py | 278 ++++++++++++- 9 files changed, 937 insertions(+), 155 deletions(-) create mode 100644 gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md create mode 100644 gitchameleon/run-config.json diff --git a/gitchameleon/README.md b/gitchameleon/README.md index ad32692..522c910 100644 --- a/gitchameleon/README.md +++ b/gitchameleon/README.md @@ -31,8 +31,9 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, - **Arms** (DG-ADR-0001 single-variable pattern; held-constant answering model): `no_grounding` / `rac` (live-decision retrieval over the example's - corpus via the shared harness runner — rac strictly as an external CLI) / - `naive_rag` (refuses until its embedder is pinned at funded-run time). + corpus via the shared harness runner — As Decided strictly as an external + CLI) / `naive_rag` (embedding retrieval over the identical corpus, pinned to + `voyage-4-large`, with query/document input types and cosine ranking). - **Corpus**: `build_corpus.py` turns each problem into a RAC decision artifact — the version pin, its rationale, companion pins, and the dataset's documentation links; never the solution, function name, or @@ -50,14 +51,14 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, ## Scaffold usage (offline, no model calls) ``` -python3 fetch_dataset.py # dataset/ (gitignored) + provenance pin +python3 fetch_dataset.py --revision 799a6a33e572a07a8985914e7251f5dea54b0ac4 python3 build_corpus.py # per-example corpora under corpus-build/ -python3 run.py --dry-run # per-example, per-arm prompt bundles +python3 run.py --dry-run # no_grounding + rac bundles python3 run.py --dry-run \ --dataset fixtures/sample_problems.json # the same, offline from the fixtures ``` -`rac` must be on `PATH` for the rac arm (external CLI only — no engine +`decided` must be on `PATH` for the rac arm (external CLI only — no engine imports, DG-ADR-0001). ## The funded run (GCB-ADR-0002 — the resolution co-primary pipeline) @@ -66,14 +67,25 @@ The pre-registered analysis and falsifier for this outcome live in `../decisiongrounding/spec/analysis-plan-amendment-1.md` (H2). Each step is resumable in isolation: -1. Pin the answering model (decisiongrounding pins `claude-opus-4-8`) and the - `naive_rag` embedder (`voyage:voyage-4-large` is the published strong - baseline there). -2. Answer every dry-run bundle: +1. Build all three arm bundles. The answering model is pinned to + `claude-opus-4-8`; the baseline embedder is pinned to + `voyage-4-large` (GCB-ADR-0003). Every frozen input is recorded in + [`run-config.json`](run-config.json): ``` + VOYAGE_API_KEY=... python3 run.py --dry-run \ + --arms no_grounding,rac,naive_rag --out out/bundles.jsonl + ``` + +2. Run a one-example-per-arm shakedown, inspect it, then answer every bundle. + `--resume` appends only missing example IDs, flushes every completion, and + records hashes of the exact prompt and grounding injected into the model: + + ``` + python3 run.py solutions --bundles out/bundles.jsonl \ + --answering claude --seed 0 --limit 1 --out out/shakedown python3 run.py solutions --bundles out/bundles.jsonl \ - --answering claude --seed 0 --out out/solutions + --answering claude --seed 0 --resume --out out/solutions ``` writes `out/solutions/solutions-.jsonl` in exactly the upstream @@ -81,9 +93,11 @@ resumable in isolation: ignored by upstream). `--answering offline-stub` exercises the plumbing keylessly; `litellm:` targets an OpenAI-compatible gateway. 3. Score each arm's file with the upstream harness — its executable tests - are the scorer; we add nothing. Clone GitChameleonBenchmark at a recorded - commit and run `evaluate --solution-path out/solutions/solutions-.jsonl` - (Docker; budget the per-version dependency installs). It writes + are the scorer; we add nothing. Clone GitChameleonBenchmark at commit + `3a1b6045a6b2a276bd24d715589cb041f8eccb93`, build its Docker image locally + from that checkout, and run + `evaluate --solution-path out/solutions/solutions-.jsonl` (budget the + per-version dependency installs). It writes `solutions-_eval_results.csv` next to each solution file. 4. Normalize the verdicts into paired resolution records (`schema/resolution_record.schema.json`) and run the pre-registered @@ -95,7 +109,8 @@ resumable in isolation: --answering-model claude-opus-4-8 --upstream-harness \ --out out/resolution_records.jsonl python3 run.py score --arm no_grounding --eval-results … --append … - python3 run.py stats --records out/resolution_records.jsonl + python3 run.py stats --records out/resolution_records.jsonl \ + --require-arms no_grounding,rac,naive_rag ``` 5. Publish the records, per-arm pass rates, and stats with the dataset @@ -105,4 +120,5 @@ resumable in isolation: ## Local decisions -See [`decisions/`](decisions/) for benchmark-local design records. +See [`decisions/`](decisions/) for benchmark-local design records, including +the funded-run execution contract and pinning rationale (GCB-ADR-0003). diff --git a/gitchameleon/answer.py b/gitchameleon/answer.py index ea6736d..07d4721 100644 --- a/gitchameleon/answer.py +++ b/gitchameleon/answer.py @@ -21,6 +21,8 @@ import json import os +import time +import urllib.error import urllib.request # The held-constant instruction every arm shares; the task prompt itself is @@ -32,6 +34,7 @@ PINNED_CLAUDE = "claude-opus-4-8" MAX_TOKENS = 2048 +MAX_ATTEMPTS = 5 def _compose_user_prompt(prompt: str, grounding: list[str]) -> str: @@ -67,7 +70,9 @@ class ClaudeModel: version = PINNED_CLAUDE def __init__(self, seed: int = 0): - self.seed = seed # recorded for provenance; the pinned model rejects sampling seeds + self.seed = ( + seed # recorded for provenance; the pinned model rejects sampling seeds + ) if not os.environ.get("ANTHROPIC_API_KEY"): raise SystemExit("the claude answering model needs ANTHROPIC_API_KEY") self._client = None @@ -80,13 +85,36 @@ def _ensure_client(self): return self._client def complete(self, prompt: str, grounding: list[str]) -> str: - msg = self._ensure_client().messages.create( - model=self.version, - max_tokens=MAX_TOKENS, - system=SYSTEM, - messages=[{"role": "user", "content": _compose_user_prompt(prompt, grounding)}], + import anthropic + + retryable = ( + anthropic.RateLimitError, + anthropic.APIConnectionError, + anthropic.InternalServerError, ) - return "".join(b.text for b in msg.content if getattr(b, "type", "") == "text") + for attempt in range(MAX_ATTEMPTS): + try: + msg = self._ensure_client().messages.create( + model=self.version, + max_tokens=MAX_TOKENS, + system=SYSTEM, + messages=[ + { + "role": "user", + "content": _compose_user_prompt(prompt, grounding), + } + ], + ) + return "".join( + block.text + for block in msg.content + if getattr(block, "type", "") == "text" + ) + except retryable: + if attempt + 1 == MAX_ATTEMPTS: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") class LiteLLMModel: @@ -112,7 +140,10 @@ def complete(self, prompt: str, grounding: list[str]) -> str: "max_tokens": MAX_TOKENS, "messages": [ {"role": "system", "content": SYSTEM}, - {"role": "user", "content": _compose_user_prompt(prompt, grounding)}, + { + "role": "user", + "content": _compose_user_prompt(prompt, grounding), + }, ], } ).encode("utf-8") @@ -124,9 +155,23 @@ def complete(self, prompt: str, grounding: list[str]) -> str: "Authorization": f"Bearer {self._key}", }, ) - with urllib.request.urlopen(req, timeout=300) as resp: - payload = json.loads(resp.read().decode("utf-8")) - return payload["choices"][0]["message"]["content"] + for attempt in range(MAX_ATTEMPTS): + try: + with urllib.request.urlopen(req, timeout=300) as resp: + payload = json.loads(resp.read().decode("utf-8")) + return payload["choices"][0]["message"]["content"] + except urllib.error.HTTPError as exc: + retryable = exc.code == 429 or 500 <= exc.code < 600 + if not retryable or attempt + 1 == MAX_ATTEMPTS: + raise + retry_after = exc.headers.get("Retry-After") + delay = float(retry_after) if retry_after else min(2**attempt, 30) + time.sleep(delay) + except urllib.error.URLError: + if attempt + 1 == MAX_ATTEMPTS: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") def make_answering_model(name: str, seed: int = 0): diff --git a/gitchameleon/arms.py b/gitchameleon/arms.py index 98e6015..3d271f7 100644 --- a/gitchameleon/arms.py +++ b/gitchameleon/arms.py @@ -16,6 +16,13 @@ from __future__ import annotations +import hashlib +import json +import math +import os +import time +import urllib.error +import urllib.request from pathlib import Path from harness.runner import RacRunner @@ -23,6 +30,89 @@ ARMS = ("no_grounding", "rac", "naive_rag") # How many retrieved decisions the rac arm feeds the answering model. RAC_TOP_K = 3 +NAIVE_RAG_MODEL = "voyage-4-large" +VOYAGE_EMBEDDINGS_URL = "https://api.voyageai.com/v1/embeddings" + + +class VoyageEmbedder: + """Pinned, retrieval-only baseline used by the funded naive_rag arm. + + Embeddings are cached by exact text within one run. This matters because + deterministic distractors recur across per-example corpora; repeated text + must not create repeated spend or a subtly different baseline. + """ + + model = NAIVE_RAG_MODEL + + def __init__(self, api_key: str | None = None, *, max_attempts: int = 5): + self.api_key = api_key or os.environ.get("VOYAGE_API_KEY") + if not self.api_key: + raise ValueError("the naive_rag arm needs VOYAGE_API_KEY") + self.max_attempts = max_attempts + self._cache: dict[tuple[str, str], list[float]] = {} + + def _request(self, texts: list[str], input_type: str) -> list[list[float]]: + body = json.dumps( + { + "input": texts, + "model": self.model, + "input_type": input_type, + "truncation": False, + } + ).encode("utf-8") + request = urllib.request.Request( + VOYAGE_EMBEDDINGS_URL, + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + }, + ) + for attempt in range(self.max_attempts): + try: + with urllib.request.urlopen(request, timeout=300) as response: + payload = json.loads(response.read().decode("utf-8")) + vectors = [entry["embedding"] for entry in payload["data"]] + if len(vectors) != len(texts): + raise RuntimeError( + f"Voyage returned {len(vectors)} vectors for {len(texts)} texts" + ) + return vectors + except urllib.error.HTTPError as exc: + retryable = exc.code == 429 or 500 <= exc.code < 600 + if not retryable or attempt + 1 == self.max_attempts: + raise + retry_after = exc.headers.get("Retry-After") + delay = float(retry_after) if retry_after else min(2**attempt, 30) + time.sleep(delay) + except urllib.error.URLError: + if attempt + 1 == self.max_attempts: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") + + def embed(self, texts: list[str], input_type: str) -> list[list[float]]: + keys = [ + (input_type, hashlib.sha256(text.encode("utf-8")).hexdigest()) + for text in texts + ] + missing: list[tuple[tuple[str, str], str]] = [ + (key, text) for key, text in zip(keys, texts) if key not in self._cache + ] + if missing: + vectors = self._request([text for _, text in missing], input_type) + for (key, _), vector in zip(missing, vectors): + self._cache[key] = vector + return [self._cache[key] for key in keys] + + +def _cosine(left: list[float], right: list[float]) -> float: + if len(left) != len(right): + raise ValueError("embedding dimensions differ") + numerator = sum(a * b for a, b in zip(left, right)) + left_norm = math.sqrt(sum(value * value for value in left)) + right_norm = math.sqrt(sum(value * value for value in right)) + return numerator / (left_norm * right_norm) if left_norm and right_norm else 0.0 def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: @@ -42,8 +132,33 @@ def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: return grounding +def naive_rag_grounding( + embedder: VoyageEmbedder, corpus_dir: Path, row: dict +) -> list[str]: + """Embedding retrieval over the exact corpus presented to RAC. + + Files are ranked by cosine similarity, then path for deterministic ties. + Query and document input types follow Voyage's retrieval contract. + """ + paths = sorted(corpus_dir.glob("*.md")) + documents = [path.read_text(encoding="utf-8") for path in paths] + if not documents: + return [] + document_vectors = embedder.embed(documents, "document") + query_vector = embedder.embed([f"{row['library']} version pin"], "query")[0] + ranked = sorted( + zip(paths, documents, document_vectors), + key=lambda item: (-_cosine(query_vector, item[2]), str(item[0])), + ) + return [document for _, document, _ in ranked[:RAC_TOP_K]] + + def assemble_grounding( - arm: str, runner: RacRunner | None, corpus_dir: Path, row: dict + arm: str, + runner: RacRunner | None, + corpus_dir: Path, + row: dict, + embedder: VoyageEmbedder | None = None, ) -> list[str]: """The grounding context one arm supplies for one example.""" if arm == "no_grounding": @@ -53,11 +168,9 @@ def assemble_grounding( raise ValueError("the rac arm needs a RacRunner (rac CLI on PATH)") return rac_grounding(runner, corpus_dir, row) if arm == "naive_rag": - raise NotImplementedError( - "naive_rag is a funded-run seam: pin the embedder there " - "(decisiongrounding pins voyage:voyage-4-large as the strong " - "published baseline) rather than shipping a weak lexical stand-in" - ) + if embedder is None: + raise ValueError("the naive_rag arm needs the pinned Voyage embedder") + return naive_rag_grounding(embedder, corpus_dir, row) raise ValueError(f"unknown arm {arm!r}; expected one of {ARMS}") diff --git a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md index af2ed6a..1ecda1a 100644 --- a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md +++ b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md @@ -37,7 +37,9 @@ prompt would measure prompt engineering, not grounding. The dataset is fetched on demand (stdlib script, provenance and content hash recorded), never vendored; three verbatim rows are committed as MIT-licensed -test fixtures with attribution. +test fixtures with attribution. The funded-run baseline is pinned by +GCB-ADR-0003 to Voyage `voyage-4-large`; it ranks the same Markdown artifacts +with explicit query/document embedding modes and deterministic cosine ties. ## Consequences @@ -45,9 +47,9 @@ Grounding quality becomes the only manipulated variable, so a rac-vs- no-grounding delta on the upstream pass rate is attributable to retrieval of the governing pin. The deliberate cost: our numbers are **not comparable to the upstream leaderboard**, which conditions the prompt on the version -explicitly — every published result must say so. The naive_rag arm refuses to -run until its embedder is pinned at funded-run time, so a weak stand-in can -never masquerade as the RAG baseline. +explicitly — every published result must say so. The naive_rag arm refuses +without the pinned embedder's API key, so a weak lexical stand-in can never +masquerade as the RAG baseline. ## Category diff --git a/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md b/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md new file mode 100644 index 0000000..f5e2b4c --- /dev/null +++ b/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md @@ -0,0 +1,63 @@ +--- +schema_version: 1 +id: GCB-7M4VX2QK9D6H +type: decision +tags: [benchmark, execution, models, provenance] +--- +# GCB-ADR-0003: Pin and Checkpoint the Funded Evidence Run + +## Status + +Accepted + +## Context + +The GitChameleon evidence run makes 328 answering calls per arm and must +survive rate limits, transient provider failures, and operator interruption +without silently changing models or duplicating spend. GCB-ADR-0001 requires +a strong embedding baseline over the same corpus as As Decided, while +GCB-ADR-0002 requires paired, provenance-bearing resolution records. + +## Decision + +Pin the held-constant answering model to `claude-opus-4-8` and the naive RAG +embedder to `voyage-4-large`. Voyage receives the exact decision artifacts +with `input_type=document` and the shared ` version pin` query with +`input_type=query`; cosine similarity ranks the top three artifacts, with +path order breaking exact ties. Truncation is disabled so an over-limit input +fails rather than changing an arm invisibly. + +Answer generation is checkpointed per arm. Existing output is never +overwritten implicitly: a caller must select `--resume` or `--overwrite`. +Resume skips completed example IDs, flushes every new JSONL record, and stores +SHA-256 hashes of the exact task prompt and grounding list. Provider calls +retry only rate limits, connection failures, and server errors with bounded +backoff. A funded run begins with `--limit 1` for every arm before the full +call set is authorized. + +The upstream GitChameleon commit remains a separate, mandatory scoring pin. +Neither an answering completion nor a locally normalized record is a pass; +only the upstream executable harness supplies that verdict. + +The concrete frozen values live in `../run-config.json`. The first registered +run uses dataset commit `799a6a33e572a07a8985914e7251f5dea54b0ac4` +(328 raw JSONL rows) and upstream harness commit +`3a1b6045a6b2a276bd24d715589cb041f8eccb93`. The scorer image is built locally +from that checkout; the upstream floating `latest` image is not evidence. + +## Consequences + +Interrupted runs can continue without duplicate calls, the three arms remain +auditable at the injection boundary, and the naive RAG label names a concrete +strong baseline. The evidence run still requires two owner-supplied provider +credentials and an upstream scoring environment. No result exists until all +three arm files are scored and the paired records pass completeness checks. + +## Category + +Process + +## Related Decisions + +- GCB-329CD3DAMG8Y +- GCB-KWRRD0T8K2Z9 diff --git a/gitchameleon/fetch_dataset.py b/gitchameleon/fetch_dataset.py index 39e0d24..e0e33e7 100644 --- a/gitchameleon/fetch_dataset.py +++ b/gitchameleon/fetch_dataset.py @@ -4,8 +4,8 @@ Downloads all rows of the upstream dataset — `cabbage972/GitChameleon-2.0` (MIT), the dataset behind GitChameleon 2.0 (arXiv:2507.12367; upstream code -Apache-2.0 at github.com/mrcabbage972/GitChameleonBenchmark) — via the -HuggingFace datasets-server API, stdlib only. Writes: +Apache-2.0 at github.com/mrcabbage972/GitChameleonBenchmark) — from the raw +JSONL at an exact Hugging Face commit, stdlib only. Writes: - ``dataset/problems.json`` — the rows, exactly as served. - ``dataset/provenance.json`` — the dataset revision, row count, retrieval @@ -18,18 +18,18 @@ from __future__ import annotations +import argparse import hashlib import json import sys import urllib.request from datetime import UTC, datetime from pathlib import Path +from urllib.parse import quote DATASET = "cabbage972/GitChameleon-2.0" CONFIG = "problems" SPLIT = "train" -PAGE = 100 -BASE = "https://datasets-server.huggingface.co" OUT_DIR = Path(__file__).resolve().parent / "dataset" @@ -38,36 +38,51 @@ def _get_json(url: str) -> dict: return json.load(response) -def fetch_rows() -> list[dict]: - rows: list[dict] = [] - offset = 0 - while True: - payload = _get_json( - f"{BASE}/rows?dataset={DATASET}&config={CONFIG}&split={SPLIT}" - f"&offset={offset}&length={PAGE}" - ) - batch = [entry["row"] for entry in payload["rows"]] - if not batch: - break - rows.extend(batch) - offset += len(batch) - if len(batch) < PAGE: - break - return rows +def fetch_rows(revision: str) -> list[dict]: + """Fetch the immutable raw JSONL at one exact Hub commit.""" + dataset_path = quote(DATASET, safe="/") + revision_path = quote(revision, safe="") + url = ( + f"https://huggingface.co/datasets/{dataset_path}/resolve/" + f"{revision_path}/dataset.jsonl" + ) + with urllib.request.urlopen(url) as response: + return [ + json.loads(line) + for line in response.read().decode("utf-8").splitlines() + if line.strip() + ] def dataset_revision() -> str: - payload = _get_json(f"https://huggingface.co/api/datasets/{DATASET.replace('/', '%2F')}") + # Hugging Face's repository endpoint treats owner/name as path segments; + # percent-encoding the slash returns HTTP 400. + payload = _get_json( + f"https://huggingface.co/api/datasets/{quote(DATASET, safe='/')}" + ) return str(payload.get("sha", "unknown")) def rows_hash(rows: list[dict]) -> str: - canonical = json.dumps(rows, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + canonical = json.dumps( + rows, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() -def main() -> int: - rows = fetch_rows() +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--revision", + default=None, + help="exact Hugging Face dataset commit (default: resolve current HEAD once)", + ) + args = parser.parse_args(argv) + revision = args.revision or dataset_revision() + if revision == "unknown": + print("fetch_dataset: upstream returned no dataset revision", file=sys.stderr) + return 1 + rows = fetch_rows(revision) if not rows: print("fetch_dataset: upstream returned no rows", file=sys.stderr) return 1 @@ -80,7 +95,7 @@ def main() -> int: "config": CONFIG, "split": SPLIT, "license": "MIT", - "revision": dataset_revision(), + "revision": revision, "n_rows": len(rows), "rows_hash": rows_hash(rows), "fetched_at": datetime.now(UTC).isoformat(), diff --git a/gitchameleon/run-config.json b/gitchameleon/run-config.json new file mode 100644 index 0000000..7ff4ec9 --- /dev/null +++ b/gitchameleon/run-config.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "preregistered-not-run", + "arms": [ + "no_grounding", + "rac", + "naive_rag" + ], + "seed": 0, + "dataset": { + "id": "cabbage972/GitChameleon-2.0", + "revision": "799a6a33e572a07a8985914e7251f5dea54b0ac4", + "rows": 328, + "rows_hash": "sha256:5fb3b2be20f180be0842950ad96d5c19a92b9cf96c45eb2896addc8c0479ece3", + "source_file": "dataset.jsonl" + }, + "answering": { + "provider": "anthropic", + "model": "claude-opus-4-8", + "max_tokens": 2048 + }, + "naive_rag": { + "provider": "voyage", + "model": "voyage-4-large", + "top_k": 3, + "similarity": "cosine", + "document_input_type": "document", + "query_input_type": "query", + "truncation": false + }, + "rac": { + "executable": "decided", + "minimum_version": "0.26.2", + "top_k": 3 + }, + "upstream_harness": { + "repository": "https://github.com/mrcabbage972/GitChameleonBenchmark", + "commit": "3a1b6045a6b2a276bd24d715589cb041f8eccb93", + "execution": "build-local-docker-image-from-pinned-commit" + }, + "protocol": { + "prompt_restates_version": false, + "upstream_leaderboard_comparable": false, + "merge_gate": false + } +} diff --git a/gitchameleon/run.py b/gitchameleon/run.py index 8207322..58b52c4 100644 --- a/gitchameleon/run.py +++ b/gitchameleon/run.py @@ -31,6 +31,7 @@ import argparse import csv +import hashlib import json import sys from pathlib import Path @@ -38,9 +39,9 @@ BENCHMARK_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(BENCHMARK_DIR.parent)) -from harness.runner import RacRunner # noqa: E402 +import arms as arms_mod -import arms as arms_mod # noqa: E402 +from harness.runner import RacRunner EXIT_OK = 0 EXIT_USAGE = 2 @@ -57,8 +58,22 @@ def _dataset_revision(provenance_path: Path) -> str | None: return None -def dry_run(rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: Path) -> int: +def _prompt_hash(bundle: dict) -> str: + return hashlib.sha256(bundle["prompt"].encode("utf-8")).hexdigest() + + +def _grounding_hash(bundle: dict) -> str: + grounding = bundle.get("grounding") or [] + return hashlib.sha256( + json.dumps(grounding, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def dry_run( + rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: Path +) -> int: runner = RacRunner() if "rac" in arm_names else None + embedder = arms_mod.VoyageEmbedder() if "naive_rag" in arm_names else None bundles = 0 with out_path.open("w", encoding="utf-8") as out: for row in rows: @@ -71,7 +86,9 @@ def dry_run(rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: ) return EXIT_USAGE for arm in arm_names: - grounding = arms_mod.assemble_grounding(arm, runner, corpus_dir, row) + grounding = arms_mod.assemble_grounding( + arm, runner, corpus_dir, row, embedder=embedder + ) out.write( json.dumps( { @@ -97,49 +114,162 @@ def cmd_solutions(args) -> int: bundles_path = Path(args.bundles) if not bundles_path.is_file(): - print(f"gitchameleon: bundles not found: {bundles_path} — run --dry-run first", - file=sys.stderr) + print( + f"gitchameleon: bundles not found: {bundles_path} — run --dry-run first", + file=sys.stderr, + ) + return EXIT_USAGE + if args.resume and args.overwrite: + print( + "gitchameleon: --resume and --overwrite are mutually exclusive", + file=sys.stderr, + ) + return EXIT_USAGE + requested_arms = [arm.strip() for arm in args.arms.split(",") if arm.strip()] + unknown = [arm for arm in requested_arms if arm not in arms_mod.ARMS] + if unknown: + print( + f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", + file=sys.stderr, + ) + return EXIT_USAGE + + bundles = [ + json.loads(line) + for line in bundles_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + bundles = [bundle for bundle in bundles if bundle["arm"] in requested_arms] + if args.limit is not None: + per_arm: dict[str, int] = {} + limited: list[dict] = [] + for bundle in bundles: + arm = bundle["arm"] + if per_arm.get(arm, 0) >= args.limit: + continue + limited.append(bundle) + per_arm[arm] = per_arm.get(arm, 0) + 1 + bundles = limited + if not bundles: + print("gitchameleon: no bundles selected", file=sys.stderr) return EXIT_USAGE + model = make_answering_model(args.answering, args.seed) revision = _dataset_revision(Path(args.provenance)) if revision is None: - print(f"gitchameleon: warning — no dataset provenance at {args.provenance}; " - "the records will carry dataset_revision=null", file=sys.stderr) + print( + f"gitchameleon: warning — no dataset provenance at {args.provenance}; " + "the records will carry dataset_revision=null", + file=sys.stderr, + ) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) - handles: dict[str, object] = {} - counts: dict[str, int] = {} - try: - with bundles_path.open(encoding="utf-8") as fp: - for line in fp: + selected_arms = sorted({bundle["arm"] for bundle in bundles}) + bundle_by_key = { + (bundle["arm"], str(bundle["example_id"])): bundle for bundle in bundles + } + completed: dict[str, set[str]] = {arm: set() for arm in selected_arms} + for arm in selected_arms: + path = out_dir / f"solutions-{arm}.jsonl" + if not path.exists(): + continue + if not args.resume and not args.overwrite: + print( + f"gitchameleon: refusing to overwrite {path}; pass --resume or --overwrite", + file=sys.stderr, + ) + return EXIT_USAGE + if args.resume: + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), 1 + ): if not line.strip(): continue - bundle = json.loads(line) - arm = bundle["arm"] - answer = model.complete(bundle["prompt"], bundle.get("grounding") or []) - if arm not in handles: - handles[arm] = (out_dir / f"solutions-{arm}.jsonl").open("w", encoding="utf-8") - # Upstream Solution model reads example_id + answer and ignores - # the extra provenance fields. - handles[arm].write(json.dumps({ - "example_id": str(bundle["example_id"]), - "answer": answer, - "arm": arm, - "library": bundle.get("library"), - "version": bundle.get("version"), + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"gitchameleon: invalid resume record {path}:{line_number}: {exc}", + file=sys.stderr, + ) + return EXIT_USAGE + record_key = (arm, str(record["example_id"])) + bundle = bundle_by_key.get(record_key) + expected = { "answering_model": model.version, "seed": args.seed, "dataset_revision": revision, - }, ensure_ascii=False) + "\n") - counts[arm] = counts.get(arm, 0) + 1 + "prompt_sha256": _prompt_hash(bundle) if bundle else None, + "grounding_sha256": _grounding_hash(bundle) if bundle else None, + } + mismatched = { + field: {"found": record.get(field), "expected": value} + for field, value in expected.items() + if record.get(field) != value + } + if mismatched: + print( + f"gitchameleon: cannot resume incompatible record " + f"{path}:{line_number}: {mismatched}", + file=sys.stderr, + ) + return EXIT_USAGE + completed[arm].add(str(record["example_id"])) + + handles: dict[str, object] = {} + counts: dict[str, int] = {} + skipped: dict[str, int] = {} + try: + for bundle in bundles: + arm = bundle["arm"] + example_id = str(bundle["example_id"]) + if example_id in completed[arm]: + skipped[arm] = skipped.get(arm, 0) + 1 + continue + grounding = bundle.get("grounding") or [] + answer = model.complete(bundle["prompt"], grounding) + if arm not in handles: + mode = "a" if args.resume else "w" + handles[arm] = (out_dir / f"solutions-{arm}.jsonl").open( + mode, encoding="utf-8" + ) + # Upstream Solution reads example_id + answer and ignores extras. + handles[arm].write( + json.dumps( + { + "example_id": example_id, + "answer": answer, + "arm": arm, + "library": bundle.get("library"), + "version": bundle.get("version"), + "answering_model": model.version, + "seed": args.seed, + "dataset_revision": revision, + "prompt_sha256": _prompt_hash(bundle), + "grounding_sha256": _grounding_hash(bundle), + }, + ensure_ascii=False, + ) + + "\n" + ) + handles[arm].flush() + counts[arm] = counts.get(arm, 0) + 1 + print(f"answered {arm} example {example_id}", file=sys.stderr) finally: for h in handles.values(): h.close() - for arm, n in sorted(counts.items()): - print(f"wrote {n} solutions -> {out_dir / f'solutions-{arm}.jsonl'}") - print("score each file with the upstream harness: " - "evaluate --solution-path (GitChameleonBenchmark)") + for arm in selected_arms: + n = counts.get(arm, 0) + already = skipped.get(arm, 0) + print( + f"wrote {n} solutions ({already} resumed) -> " + f"{out_dir / f'solutions-{arm}.jsonl'}" + ) + print( + "score each file with the upstream harness: " + "evaluate --solution-path (GitChameleonBenchmark)" + ) return EXIT_OK @@ -162,14 +292,19 @@ def cmd_score(args) -> int: "example_id": eid, "arm": args.arm, "passed": _truthy(row.get("passed", "")), - "compiled": _truthy(row["compiled"]) if row.get("compiled") not in (None, "") else None, + "compiled": _truthy(row["compiled"]) + if row.get("compiled") not in (None, "") + else None, "answering_model": args.answering_model, "seed": args.seed, - "dataset_revision": args.dataset_revision or _dataset_revision(Path(args.provenance)), + "dataset_revision": args.dataset_revision + or _dataset_revision(Path(args.provenance)), "upstream_harness": args.upstream_harness, } if not by_example: - print("gitchameleon: no solution_code rows found in the eval CSV", file=sys.stderr) + print( + "gitchameleon: no solution_code rows found in the eval CSV", file=sys.stderr + ) return EXIT_USAGE out_path = Path(args.out) out_path.parent.mkdir(parents=True, exist_ok=True) @@ -178,8 +313,10 @@ def cmd_score(args) -> int: for eid in sorted(by_example, key=int): out.write(json.dumps(by_example[eid], ensure_ascii=False) + "\n") passed = sum(1 for r in by_example.values() if r["passed"]) - print(f"{args.arm}: {passed}/{len(by_example)} passed -> {out_path}" - f" ({'appended' if args.append else 'wrote'})") + print( + f"{args.arm}: {passed}/{len(by_example)} passed -> {out_path}" + f" ({'appended' if args.append else 'wrote'})" + ) return EXIT_OK @@ -190,10 +327,43 @@ def cmd_stats(args) -> int: print(f"gitchameleon: records not found: {records_path}", file=sys.stderr) return EXIT_USAGE sys.path.insert(0, str(BENCHMARK_DIR.parent / "decisiongrounding")) - from scoring.stats import paired_significance # noqa: E402 - - records = [json.loads(line) for line in records_path.read_text(encoding="utf-8").splitlines() - if line.strip()] + from scoring.stats import paired_significance + + records = [ + json.loads(line) + for line in records_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + required_arms = [arm.strip() for arm in args.require_arms.split(",") if arm.strip()] + by_arm: dict[str, set[str]] = {} + seen: set[tuple[str, str, object]] = set() + for record in records: + key = (record["arm"], str(record["example_id"]), record.get("seed")) + if key in seen: + print(f"gitchameleon: duplicate resolution record {key}", file=sys.stderr) + return EXIT_USAGE + seen.add(key) + by_arm.setdefault(record["arm"], set()).add(str(record["example_id"])) + missing_arms = [arm for arm in required_arms if arm not in by_arm] + if missing_arms: + print( + f"gitchameleon: required arm(s) missing from records: {missing_arms}", + file=sys.stderr, + ) + return EXIT_USAGE + compared = required_arms or sorted(by_arm) + if len(compared) < 2: + print("gitchameleon: stats needs at least two arms", file=sys.stderr) + return EXIT_USAGE + expected = by_arm[compared[0]] + incomplete = { + arm: sorted(expected.symmetric_difference(by_arm[arm]), key=int) + for arm in compared[1:] + if by_arm[arm] != expected + } + if incomplete: + print(f"gitchameleon: incomplete paired records: {incomplete}", file=sys.stderr) + return EXIT_USAGE out = paired_significance(records, outcome="passed") rendered = json.dumps(out, indent=2) if args.out: @@ -206,60 +376,131 @@ def cmd_stats(args) -> int: def _add_common_provenance(sub) -> None: - sub.add_argument("--provenance", default=str(BENCHMARK_DIR / "dataset" / "provenance.json"), - help="fetch_dataset.py provenance pin (for dataset_revision).") + sub.add_argument( + "--provenance", + default=str(BENCHMARK_DIR / "dataset" / "provenance.json"), + help="fetch_dataset.py provenance pin (for dataset_revision).", + ) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="gitchameleon", description="GitChameleon evidence run: dry-run offline; " - "solutions/score/stats are the funded-run pipeline.", + "solutions/score/stats are the funded-run pipeline.", ) sub = parser.add_subparsers(dest="mode") - p_sol = sub.add_parser("solutions", help="answer each bundle; write per-arm solution JSONL") - p_sol.add_argument("--bundles", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl")) - p_sol.add_argument("--answering", default="offline-stub", - help="offline-stub | claude | litellm:") + p_sol = sub.add_parser( + "solutions", help="answer each bundle; write per-arm solution JSONL" + ) + p_sol.add_argument( + "--bundles", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl") + ) + p_sol.add_argument( + "--answering", + default="offline-stub", + help="offline-stub | claude | litellm:", + ) + p_sol.add_argument( + "--arms", + default=",".join(arms_mod.ARMS), + help="comma-separated subset of arms to answer", + ) + p_sol.add_argument( + "--limit", + type=int, + default=None, + help="answer at most N examples per selected arm (shakedowns only)", + ) + p_sol.add_argument( + "--resume", + action="store_true", + help="append only missing example ids to existing arm files", + ) + p_sol.add_argument( + "--overwrite", + action="store_true", + help="replace existing selected-arm solution files", + ) p_sol.add_argument("--seed", type=int, default=0) p_sol.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "solutions")) _add_common_provenance(p_sol) p_sol.set_defaults(func=cmd_solutions) - p_score = sub.add_parser("score", help="normalize an upstream eval_results CSV " - "into paired resolution records") + p_score = sub.add_parser( + "score", + help="normalize an upstream eval_results CSV into paired resolution records", + ) p_score.add_argument("--arm", required=True, choices=arms_mod.ARMS) - p_score.add_argument("--eval-results", required=True, - help="the upstream harness's *_eval_results.csv for this arm") - p_score.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl")) - p_score.add_argument("--append", action="store_true", - help="append to --out (one file across arms)") - p_score.add_argument("--answering-model", default=None, - help="the pinned answering model the solutions used") + p_score.add_argument( + "--eval-results", + required=True, + help="the upstream harness's *_eval_results.csv for this arm", + ) + p_score.add_argument( + "--out", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl") + ) + p_score.add_argument( + "--append", action="store_true", help="append to --out (one file across arms)" + ) + p_score.add_argument( + "--answering-model", + default=None, + help="the pinned answering model the solutions used", + ) p_score.add_argument("--seed", type=int, default=0) p_score.add_argument("--dataset-revision", default=None) - p_score.add_argument("--upstream-harness", default=None, - help="GitChameleonBenchmark commit/tag used to score") + p_score.add_argument( + "--upstream-harness", + default=None, + help="GitChameleonBenchmark commit/tag used to score", + ) _add_common_provenance(p_score) p_score.set_defaults(func=cmd_score) - p_stats = sub.add_parser("stats", help="paired significance over resolution records") - p_stats.add_argument("--records", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl")) - p_stats.add_argument("--out", default=None, help="write JSON here instead of stdout") + p_stats = sub.add_parser( + "stats", help="paired significance over resolution records" + ) + p_stats.add_argument( + "--records", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl") + ) + p_stats.add_argument( + "--out", default=None, help="write JSON here instead of stdout" + ) + p_stats.add_argument( + "--require-arms", + default="", + help="comma-separated arms that must all contain the same example IDs", + ) p_stats.set_defaults(func=cmd_stats) # Legacy top-level dry-run flags (the offline scaffold surface). - parser.add_argument("--dry-run", action="store_true", - help="Assemble per-example, per-arm prompt bundles without any model call.") - parser.add_argument("--dataset", default=str(BENCHMARK_DIR / "dataset" / "problems.json"), - help="Problem rows (fetch_dataset.py output, or the committed fixture).") - parser.add_argument("--corpus", default=str(BENCHMARK_DIR / "corpus-build"), - help="Per-example corpus root (build_corpus.py output).") - parser.add_argument("--arms", default="no_grounding,rac", - help="Comma-separated arms (naive_rag refuses until its embedder is pinned).") - parser.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl"), - help="Where the dry-run writes the prompt bundles (JSONL).") + parser.add_argument( + "--dry-run", + action="store_true", + help="Assemble per-example, per-arm prompt bundles without any model call.", + ) + parser.add_argument( + "--dataset", + default=str(BENCHMARK_DIR / "dataset" / "problems.json"), + help="Problem rows (fetch_dataset.py output, or the committed fixture).", + ) + parser.add_argument( + "--corpus", + default=str(BENCHMARK_DIR / "corpus-build"), + help="Per-example corpus root (build_corpus.py output).", + ) + parser.add_argument( + "--arms", + default="no_grounding,rac", + help="Comma-separated arms (naive_rag refuses until its embedder is pinned).", + ) + parser.add_argument( + "--out", + default=str(BENCHMARK_DIR / "out" / "bundles.jsonl"), + help="Where the dry-run writes the prompt bundles (JSONL).", + ) args = parser.parse_args(argv) if getattr(args, "mode", None): @@ -268,7 +509,10 @@ def main(argv: list[str] | None = None) -> int: arm_names = [arm.strip() for arm in args.arms.split(",") if arm.strip()] unknown = [arm for arm in arm_names if arm not in arms_mod.ARMS] if unknown: - print(f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", file=sys.stderr) + print( + f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", + file=sys.stderr, + ) return EXIT_USAGE dataset_path = Path(args.dataset) @@ -293,7 +537,7 @@ def main(argv: list[str] | None = None) -> int: out_path.parent.mkdir(parents=True, exist_ok=True) try: return dry_run(load_rows(dataset_path), Path(args.corpus), arm_names, out_path) - except NotImplementedError as exc: + except (NotImplementedError, ValueError) as exc: print(f"gitchameleon: {exc}", file=sys.stderr) return EXIT_USAGE diff --git a/tests/test_gitchameleon_scaffold.py b/tests/test_gitchameleon_scaffold.py index 97b879d..a07f9f2 100644 --- a/tests/test_gitchameleon_scaffold.py +++ b/tests/test_gitchameleon_scaffold.py @@ -18,6 +18,9 @@ GCB = REPO_ROOT / "gitchameleon" FIXTURES = GCB / "fixtures" / "sample_problems.json" +sys.path.insert(0, str(GCB)) +import arms as arms_mod +import fetch_dataset as fetch_mod def _run(script: str, *args: str) -> subprocess.CompletedProcess: @@ -32,7 +35,13 @@ def _run(script: str, *args: str) -> subprocess.CompletedProcess: def _build(tmp_path, name: str = "corpus"): out = tmp_path / name completed = _run( - "build_corpus.py", "--dataset", str(FIXTURES), "--out", str(out), "--distractors", "2" + "build_corpus.py", + "--dataset", + str(FIXTURES), + "--out", + str(out), + "--distractors", + "2", ) assert completed.returncode == 0, completed.stderr return out @@ -40,7 +49,8 @@ def _build(tmp_path, name: str = "corpus"): def _corpus_bytes(root) -> dict[str, bytes]: return { - str(path.relative_to(root)): path.read_bytes() for path in sorted(root.rglob("*.md")) + str(path.relative_to(root)): path.read_bytes() + for path in sorted(root.rglob("*.md")) } @@ -48,16 +58,69 @@ def _fixture_rows() -> list[dict]: return json.loads(FIXTURES.read_text(encoding="utf-8"))["rows"] +def test_funded_run_config_matches_implemented_pins(): + config = json.loads((GCB / "run-config.json").read_text(encoding="utf-8")) + assert config["status"] == "preregistered-not-run" + assert config["arms"] == list(arms_mod.ARMS) + assert config["answering"]["model"] == "claude-opus-4-8" + assert config["naive_rag"]["model"] == arms_mod.NAIVE_RAG_MODEL + assert config["naive_rag"]["top_k"] == arms_mod.RAC_TOP_K + assert len(config["dataset"]["revision"]) == 40 + assert len(config["upstream_harness"]["commit"]) == 40 + + def test_corpus_builder_is_deterministic(tmp_path): first = _corpus_bytes(_build(tmp_path, "a")) second = _corpus_bytes(_build(tmp_path, "b")) assert first == second +def test_dataset_revision_preserves_the_owner_name_path(monkeypatch): + seen = [] + + def fake_get(url): + seen.append(url) + return {"sha": "dataset-pin"} + + monkeypatch.setattr(fetch_mod, "_get_json", fake_get) + assert fetch_mod.dataset_revision() == "dataset-pin" + assert seen == ["https://huggingface.co/api/datasets/cabbage972/GitChameleon-2.0"] + + +def test_dataset_rows_are_fetched_at_the_exact_revision(monkeypatch): + seen = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_): + return None + + def read(self): + return b'{"example_id":"7"}\n' + + def fake_open(url): + seen.append(url) + return Response() + + monkeypatch.setattr(fetch_mod.urllib.request, "urlopen", fake_open) + assert fetch_mod.fetch_rows("abc123") == [{"example_id": "7"}] + assert seen == [ + ( + "https://huggingface.co/datasets/cabbage972/GitChameleon-2.0/" + "resolve/abc123/dataset.jsonl" + ) + ] + + def test_built_corpora_are_schema_valid(tmp_path): corpus = _build(tmp_path) completed = subprocess.run( - ["decided", "validate", str(corpus)], capture_output=True, text=True, check=False + ["decided", "validate", str(corpus)], + capture_output=True, + text=True, + check=False, ) assert completed.returncode == 0, completed.stdout + completed.stderr @@ -66,7 +129,9 @@ def test_decision_artifacts_never_leak_solutions_or_tests(tmp_path): corpus = _build(tmp_path) leaks = [row["solution"].strip() for row in _fixture_rows()] leaks += [row["test"].strip() for row in _fixture_rows()] - corpus_text = "\n".join(text.decode("utf-8") for text in _corpus_bytes(corpus).values()) + corpus_text = "\n".join( + text.decode("utf-8") for text in _corpus_bytes(corpus).values() + ) for leak in leaks: assert leak not in corpus_text @@ -113,7 +178,8 @@ def test_prompt_never_states_the_pinned_version(tmp_path): assert bundle["version"] not in bundle["prompt"] -def test_naive_rag_refuses_until_embedder_is_pinned(tmp_path): +def test_naive_rag_refuses_without_the_pinned_embedder_key(tmp_path, monkeypatch): + monkeypatch.delenv("VOYAGE_API_KEY", raising=False) corpus = _build(tmp_path) completed = _run( "run.py", @@ -128,7 +194,24 @@ def test_naive_rag_refuses_until_embedder_is_pinned(tmp_path): str(tmp_path / "bundles.jsonl"), ) assert completed.returncode == 2 - assert "embedder" in completed.stderr + assert "VOYAGE_API_KEY" in completed.stderr + + +def test_naive_rag_ranks_the_governing_pin_first(tmp_path): + corpus = _build(tmp_path) + row = _fixture_rows()[0] + corpus_dir = corpus / f"example-{row['example_id']}" + + class FakeEmbedder: + def embed(self, texts, input_type): + if input_type == "query": + return [[1.0, 0.0]] + heading = f"# Library Version Pin: {row['library']} {row['version']}" + return [[1.0, 0.0] if heading in text else [0.0, 1.0] for text in texts] + + grounding = arms_mod.naive_rag_grounding(FakeEmbedder(), corpus_dir, row) + assert len(grounding) == 3 + assert f"# Library Version Pin: {row['library']} {row['version']}" in grounding[0] def test_bare_invocation_points_at_the_modes(tmp_path): @@ -147,8 +230,14 @@ def _bundles_file(tmp_path) -> str: corpus = _build(tmp_path) out = tmp_path / "bundles.jsonl" completed = _run( - "run.py", "--dry-run", "--dataset", str(FIXTURES), "--corpus", str(corpus), - "--out", str(out), + "run.py", + "--dry-run", + "--dataset", + str(FIXTURES), + "--corpus", + str(corpus), + "--out", + str(out), ) assert completed.returncode == 0, completed.stderr return str(out) @@ -156,14 +245,22 @@ def _bundles_file(tmp_path) -> str: def _solutions(tmp_path, out_name: str) -> dict[str, list[dict]]: completed = _run( - "run.py", "solutions", "--bundles", _bundles_file(tmp_path), - "--answering", "offline-stub", "--out", str(tmp_path / out_name), + "run.py", + "solutions", + "--bundles", + _bundles_file(tmp_path), + "--answering", + "offline-stub", + "--out", + str(tmp_path / out_name), ) assert completed.returncode == 0, completed.stderr out: dict[str, list[dict]] = {} for path in sorted((tmp_path / out_name).glob("solutions-*.jsonl")): arm = path.stem.removeprefix("solutions-") - out[arm] = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + out[arm] = [ + json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + ] return out @@ -179,13 +276,106 @@ def test_offline_stub_solutions_are_deterministic_and_upstream_shaped(tmp_path): assert isinstance(rec["example_id"], str) and rec["answer"] assert rec["arm"] == arm assert "offline-stub" in rec["answer"] # plumbing output is labelled + assert len(rec["prompt_sha256"]) == 64 + assert len(rec["grounding_sha256"]) == 64 + + +def test_solutions_resume_only_answers_missing_examples(tmp_path): + bundles = _bundles_file(tmp_path) + out = tmp_path / "resume-solutions" + first = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "no_grounding", + "--limit", + "1", + "--out", + str(out), + ) + assert first.returncode == 0, first.stderr + second = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "no_grounding", + "--limit", + "2", + "--resume", + "--out", + str(out), + ) + assert second.returncode == 0, second.stderr + records = [ + json.loads(line) + for line in (out / "solutions-no_grounding.jsonl").read_text().splitlines() + ] + assert len(records) == 2 + assert len({record["example_id"] for record in records}) == 2 + + +def test_solutions_resume_refuses_a_different_seed(tmp_path): + bundles = _bundles_file(tmp_path) + out = tmp_path / "incompatible-resume" + first = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "rac", + "--limit", + "1", + "--seed", + "0", + "--out", + str(out), + ) + assert first.returncode == 0, first.stderr + second = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "rac", + "--limit", + "1", + "--seed", + "1", + "--resume", + "--out", + str(out), + ) + assert second.returncode == 2 + assert "cannot resume incompatible record" in second.stderr def test_solutions_refuse_real_backends_without_keys(tmp_path, monkeypatch): - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) # the subprocess inherits os.environ + monkeypatch.delenv( + "ANTHROPIC_API_KEY", raising=False + ) # the subprocess inherits os.environ completed = _run( - "run.py", "solutions", "--bundles", _bundles_file(tmp_path), - "--answering", "claude", "--out", str(tmp_path / "sol"), + "run.py", + "solutions", + "--bundles", + _bundles_file(tmp_path), + "--answering", + "claude", + "--out", + str(tmp_path / "sol"), ) assert completed.returncode != 0 assert "ANTHROPIC_API_KEY" in (completed.stderr + completed.stdout) @@ -194,17 +384,35 @@ def test_solutions_refuse_real_backends_without_keys(tmp_path, monkeypatch): def _score_records(tmp_path) -> list[dict]: records = tmp_path / "resolution_records.jsonl" completed = _run( - "run.py", "score", "--arm", "rac", "--eval-results", str(EVAL_RAC), - "--out", str(records), "--answering-model", "claude-opus-4-8", - "--upstream-harness", "test-commit", + "run.py", + "score", + "--arm", + "rac", + "--eval-results", + str(EVAL_RAC), + "--out", + str(records), + "--answering-model", + "claude-opus-4-8", + "--upstream-harness", + "test-commit", ) assert completed.returncode == 0, completed.stderr completed = _run( - "run.py", "score", "--arm", "no_grounding", "--eval-results", str(EVAL_NONE), - "--out", str(records), "--append", + "run.py", + "score", + "--arm", + "no_grounding", + "--eval-results", + str(EVAL_NONE), + "--out", + str(records), + "--append", ) assert completed.returncode == 0, completed.stderr - return [json.loads(line) for line in records.read_text(encoding="utf-8").splitlines()] + return [ + json.loads(line) for line in records.read_text(encoding="utf-8").splitlines() + ] def test_score_emits_schema_valid_paired_records(tmp_path): @@ -233,3 +441,33 @@ def test_stats_reproduces_the_hand_computed_mcnemar(tmp_path): # exact two-sided binomial at min(2,0)=0 of 2 discordant: 2 * (1/4) = 0.5 assert pair["mcnemar"]["p_value"] == 0.5 assert pair["odds_ratio"]["degenerate"] is True + + +def test_stats_refuses_incomplete_required_arms(tmp_path): + records = tmp_path / "resolution_records.jsonl" + _score_records(tmp_path) + rac_ids = [ + json.loads(line)["example_id"] + for line in records.read_text(encoding="utf-8").splitlines() + if json.loads(line)["arm"] == "rac" + ] + removed_id = rac_ids[-1] + kept = [ + line + for line in records.read_text(encoding="utf-8").splitlines() + if not ( + json.loads(line)["arm"] == "rac" + and json.loads(line)["example_id"] == removed_id + ) + ] + records.write_text("\n".join(kept) + "\n", encoding="utf-8") + completed = _run( + "run.py", + "stats", + "--records", + str(records), + "--require-arms", + "no_grounding,rac", + ) + assert completed.returncode == 2 + assert "incomplete paired records" in completed.stderr From eec52415187a77615784720991e09f90f5194586 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sun, 2 Aug 2026 08:36:17 +0100 Subject: [PATCH 2/2] refactor(benchmarks): rename RAC arm to As Decided --- gitchameleon/README.md | 18 +++--- gitchameleon/arms.py | 24 ++++---- gitchameleon/build_corpus.py | 4 +- .../decisions/GCB-ADR-0001-benchmark-scope.md | 7 ++- ....csv => sample_eval_results_asdecided.csv} | 0 gitchameleon/run-config.json | 4 +- gitchameleon/run.py | 4 +- .../schema/resolution_record.schema.json | 2 +- tests/test_gitchameleon_scaffold.py | 55 ++++++++++++------- 9 files changed, 68 insertions(+), 50 deletions(-) rename gitchameleon/fixtures/{sample_eval_results_rac.csv => sample_eval_results_asdecided.csv} (100%) diff --git a/gitchameleon/README.md b/gitchameleon/README.md index 522c910..071bf1b 100644 --- a/gitchameleon/README.md +++ b/gitchameleon/README.md @@ -30,9 +30,9 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, ## Design (GCB-ADR-0001) - **Arms** (DG-ADR-0001 single-variable pattern; held-constant answering - model): `no_grounding` / `rac` (live-decision retrieval over the example's - corpus via the shared harness runner — As Decided strictly as an external - CLI) / `naive_rag` (embedding retrieval over the identical corpus, pinned to + model): `no_grounding` / `asdecided` (live-decision retrieval over the + example's corpus via the shared harness runner — As Decided strictly as an + external CLI) / `naive_rag` (embedding retrieval over the identical corpus, pinned to `voyage-4-large`, with query/document input types and cosine ranking). - **Corpus**: `build_corpus.py` turns each problem into a RAC decision artifact — the version pin, its rationale, companion pins, and the @@ -53,12 +53,12 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, ``` python3 fetch_dataset.py --revision 799a6a33e572a07a8985914e7251f5dea54b0ac4 python3 build_corpus.py # per-example corpora under corpus-build/ -python3 run.py --dry-run # no_grounding + rac bundles +python3 run.py --dry-run # no_grounding + asdecided bundles python3 run.py --dry-run \ --dataset fixtures/sample_problems.json # the same, offline from the fixtures ``` -`decided` must be on `PATH` for the rac arm (external CLI only — no engine +`decided` must be on `PATH` for the `asdecided` arm (external CLI only — no engine imports, DG-ADR-0001). ## The funded run (GCB-ADR-0002 — the resolution co-primary pipeline) @@ -74,7 +74,7 @@ resumable in isolation: ``` VOYAGE_API_KEY=... python3 run.py --dry-run \ - --arms no_grounding,rac,naive_rag --out out/bundles.jsonl + --arms no_grounding,asdecided,naive_rag --out out/bundles.jsonl ``` 2. Run a one-example-per-arm shakedown, inspect it, then answer every bundle. @@ -104,13 +104,13 @@ resumable in isolation: paired analysis: ``` - python3 run.py score --arm rac \ - --eval-results out/solutions/solutions-rac_eval_results.csv \ + python3 run.py score --arm asdecided \ + --eval-results out/solutions/solutions-asdecided_eval_results.csv \ --answering-model claude-opus-4-8 --upstream-harness \ --out out/resolution_records.jsonl python3 run.py score --arm no_grounding --eval-results … --append … python3 run.py stats --records out/resolution_records.jsonl \ - --require-arms no_grounding,rac,naive_rag + --require-arms no_grounding,asdecided,naive_rag ``` 5. Publish the records, per-arm pass rates, and stats with the dataset diff --git a/gitchameleon/arms.py b/gitchameleon/arms.py index 3d271f7..09b0417 100644 --- a/gitchameleon/arms.py +++ b/gitchameleon/arms.py @@ -5,8 +5,8 @@ answering model (the DG-ADR-0001 single-variable design): - ``no_grounding`` — the problem alone; the model answers from its weights. -- ``rac`` — the governing version-pin decision retrieved from the - example's corpus via the live-decision query (`rac find --decisions`), +- ``asdecided`` — the governing version-pin decision retrieved from the + example's corpus via the live-decision query (`decided find --decisions`), driven strictly as an external CLI through the shared harness runner. - ``naive_rag`` — embedding retrieval over the same corpus. Deliberately a seam: the embedder is pinned at funded-run time (mirroring @@ -27,9 +27,9 @@ from harness.runner import RacRunner -ARMS = ("no_grounding", "rac", "naive_rag") -# How many retrieved decisions the rac arm feeds the answering model. -RAC_TOP_K = 3 +ARMS = ("no_grounding", "asdecided", "naive_rag") +# Both grounded arms supply the same number of artifacts to the answering model. +GROUNDING_TOP_K = 3 NAIVE_RAG_MODEL = "voyage-4-large" VOYAGE_EMBEDDINGS_URL = "https://api.voyageai.com/v1/embeddings" @@ -115,8 +115,8 @@ def _cosine(left: list[float], right: list[float]) -> float: return numerator / (left_norm * right_norm) if left_norm and right_norm else 0.0 -def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: - """The rac arm: live-decision retrieval, retrieved artifacts verbatim. +def asdecided_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: + """The As Decided arm: live-decision retrieval, artifacts verbatim. The query is what an agent grounding itself would ask before writing code against a library: which live decisions bind this dependency? @@ -124,7 +124,7 @@ def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: query = f"{row['library']} version pin" returned = runner.find_ids(query, str(corpus_dir), decisions=True) grounding: list[str] = [] - for artifact_id in returned[:RAC_TOP_K]: + for artifact_id in returned[:GROUNDING_TOP_K]: resolved = runner.resolve(artifact_id, str(corpus_dir)) payload = resolved.payload() if resolved.exit_code == 0 and "path" in payload: @@ -150,7 +150,7 @@ def naive_rag_grounding( zip(paths, documents, document_vectors), key=lambda item: (-_cosine(query_vector, item[2]), str(item[0])), ) - return [document for _, document, _ in ranked[:RAC_TOP_K]] + return [document for _, document, _ in ranked[:GROUNDING_TOP_K]] def assemble_grounding( @@ -163,10 +163,10 @@ def assemble_grounding( """The grounding context one arm supplies for one example.""" if arm == "no_grounding": return [] - if arm == "rac": + if arm == "asdecided": if runner is None: - raise ValueError("the rac arm needs a RacRunner (rac CLI on PATH)") - return rac_grounding(runner, corpus_dir, row) + raise ValueError("the asdecided arm needs the decided CLI on PATH") + return asdecided_grounding(runner, corpus_dir, row) if arm == "naive_rag": if embedder is None: raise ValueError("the naive_rag arm needs the pinned Voyage embedder") diff --git a/gitchameleon/build_corpus.py b/gitchameleon/build_corpus.py index 3b12fc2..dad2048 100644 --- a/gitchameleon/build_corpus.py +++ b/gitchameleon/build_corpus.py @@ -6,7 +6,7 @@ respect: "this codebase targets ``==`` on Python ````". The builder writes that decision plus a deterministic set of distractor pins (other examples' decisions) into a per-example corpus -directory, so the rac arm's grounding retrieval has to find the governing pin +directory, so the As Decided arm's grounding retrieval has to find the governing pin among plausible competitors — the same distractor model decisiongrounding uses. @@ -39,7 +39,7 @@ def artifact_id(example_id: str) -> str: def decision_markdown(row: dict) -> str: - """One version-pin decision artifact, schema-valid for `rac validate`.""" + """One version-pin decision artifact, schema-valid for `decided validate`.""" library = row["library"] version = row["version"] python_version = row["python_version"] diff --git a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md index 1ecda1a..7c5bd12 100644 --- a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md +++ b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md @@ -43,7 +43,12 @@ with explicit query/document embedding modes and deterministic cosine ties. ## Consequences -Grounding quality becomes the only manipulated variable, so a rac-vs- +The product-backed arm is named `asdecided` in CLI arguments, generated files, +records, and published results. The former `rac` label is not an accepted alias: +mixing both names would make paired-run completeness ambiguous and retain a +retired product name in durable evidence. + +Grounding quality becomes the only manipulated variable, so an asdecided-vs- no-grounding delta on the upstream pass rate is attributable to retrieval of the governing pin. The deliberate cost: our numbers are **not comparable to the upstream leaderboard**, which conditions the prompt on the version diff --git a/gitchameleon/fixtures/sample_eval_results_rac.csv b/gitchameleon/fixtures/sample_eval_results_asdecided.csv similarity index 100% rename from gitchameleon/fixtures/sample_eval_results_rac.csv rename to gitchameleon/fixtures/sample_eval_results_asdecided.csv diff --git a/gitchameleon/run-config.json b/gitchameleon/run-config.json index 7ff4ec9..390eadb 100644 --- a/gitchameleon/run-config.json +++ b/gitchameleon/run-config.json @@ -3,7 +3,7 @@ "status": "preregistered-not-run", "arms": [ "no_grounding", - "rac", + "asdecided", "naive_rag" ], "seed": 0, @@ -28,7 +28,7 @@ "query_input_type": "query", "truncation": false }, - "rac": { + "asdecided": { "executable": "decided", "minimum_version": "0.26.2", "top_k": 3 diff --git a/gitchameleon/run.py b/gitchameleon/run.py index 58b52c4..c5ed59a 100644 --- a/gitchameleon/run.py +++ b/gitchameleon/run.py @@ -72,7 +72,7 @@ def _grounding_hash(bundle: dict) -> str: def dry_run( rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: Path ) -> int: - runner = RacRunner() if "rac" in arm_names else None + runner = RacRunner() if "asdecided" in arm_names else None embedder = arms_mod.VoyageEmbedder() if "naive_rag" in arm_names else None bundles = 0 with out_path.open("w", encoding="utf-8") as out: @@ -493,7 +493,7 @@ def main(argv: list[str] | None = None) -> int: ) parser.add_argument( "--arms", - default="no_grounding,rac", + default="no_grounding,asdecided", help="Comma-separated arms (naive_rag refuses until its embedder is pinned).", ) parser.add_argument( diff --git a/gitchameleon/schema/resolution_record.schema.json b/gitchameleon/schema/resolution_record.schema.json index da90e92..5fea8cb 100644 --- a/gitchameleon/schema/resolution_record.schema.json +++ b/gitchameleon/schema/resolution_record.schema.json @@ -13,7 +13,7 @@ }, "arm": { "type": "string", - "enum": ["no_grounding", "rac", "naive_rag"] + "enum": ["no_grounding", "asdecided", "naive_rag"] }, "passed": { "type": "boolean", diff --git a/tests/test_gitchameleon_scaffold.py b/tests/test_gitchameleon_scaffold.py index a07f9f2..ad7e93c 100644 --- a/tests/test_gitchameleon_scaffold.py +++ b/tests/test_gitchameleon_scaffold.py @@ -2,8 +2,8 @@ """GitChameleon evidence-run scaffold battery (offline; fixtures only). Exercises the key-less surface: the corpus builder is deterministic and emits -schema-valid RAC decisions, the dry-run assembles honest prompt bundles (the -rac arm's grounding leads with the governing pin; prompts never leak the +schema-valid As Decided decisions, the dry-run assembles honest prompt bundles +(the As Decided arm's grounding leads with the governing pin; prompts never leak the pinned version; grounding never leaks solutions or tests), and the funded-run seams refuse loudly instead of running weak stand-ins. """ @@ -64,7 +64,7 @@ def test_funded_run_config_matches_implemented_pins(): assert config["arms"] == list(arms_mod.ARMS) assert config["answering"]["model"] == "claude-opus-4-8" assert config["naive_rag"]["model"] == arms_mod.NAIVE_RAG_MODEL - assert config["naive_rag"]["top_k"] == arms_mod.RAC_TOP_K + assert config["naive_rag"]["top_k"] == arms_mod.GROUNDING_TOP_K assert len(config["dataset"]["revision"]) == 40 assert len(config["upstream_harness"]["commit"]) == 40 @@ -136,7 +136,7 @@ def test_decision_artifacts_never_leak_solutions_or_tests(tmp_path): assert leak not in corpus_text -def _dry_run_bundles(tmp_path, arms: str = "no_grounding,rac") -> list[dict]: +def _dry_run_bundles(tmp_path, arms: str = "no_grounding,asdecided") -> list[dict]: corpus = _build(tmp_path) out = tmp_path / "bundles.jsonl" completed = _run( @@ -155,11 +155,11 @@ def _dry_run_bundles(tmp_path, arms: str = "no_grounding,rac") -> list[dict]: return [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] -def test_rac_arm_grounding_leads_with_the_governing_pin(tmp_path): +def test_asdecided_arm_grounding_leads_with_the_governing_pin(tmp_path): bundles = _dry_run_bundles(tmp_path) - rac_bundles = [b for b in bundles if b["arm"] == "rac"] - assert len(rac_bundles) == len(_fixture_rows()) - for bundle in rac_bundles: + asdecided_bundles = [b for b in bundles if b["arm"] == "asdecided"] + assert len(asdecided_bundles) == len(_fixture_rows()) + for bundle in asdecided_bundles: assert bundle["grounding"], bundle["example_id"] heading = f"# Library Version Pin: {bundle['library']} {bundle['version']}" assert heading in bundle["grounding"][0] @@ -220,9 +220,22 @@ def test_bare_invocation_points_at_the_modes(tmp_path): assert "solutions / score / stats" in completed.stderr +def test_retired_rac_arm_name_is_rejected(): + completed = _run( + "run.py", + "--dry-run", + "--dataset", + str(FIXTURES), + "--arms", + "rac", + ) + assert completed.returncode == 2 + assert "unknown arm(s) ['rac']" in completed.stderr + + # --- the resolution co-primary pipeline (GCB-ADR-0002), offline --------------- -EVAL_RAC = GCB / "fixtures" / "sample_eval_results_rac.csv" +EVAL_ASDECIDED = GCB / "fixtures" / "sample_eval_results_asdecided.csv" EVAL_NONE = GCB / "fixtures" / "sample_eval_results_no_grounding.csv" @@ -268,7 +281,7 @@ def test_offline_stub_solutions_are_deterministic_and_upstream_shaped(tmp_path): first = _solutions(tmp_path, "sol-a") second = _solutions(tmp_path, "sol-b") assert first == second - assert set(first) == {"no_grounding", "rac"} + assert set(first) == {"no_grounding", "asdecided"} for arm, records in first.items(): assert len(records) == len(_fixture_rows()) for rec in records: @@ -333,7 +346,7 @@ def test_solutions_resume_refuses_a_different_seed(tmp_path): "--answering", "offline-stub", "--arms", - "rac", + "asdecided", "--limit", "1", "--seed", @@ -350,7 +363,7 @@ def test_solutions_resume_refuses_a_different_seed(tmp_path): "--answering", "offline-stub", "--arms", - "rac", + "asdecided", "--limit", "1", "--seed", @@ -387,9 +400,9 @@ def _score_records(tmp_path) -> list[dict]: "run.py", "score", "--arm", - "rac", + "asdecided", "--eval-results", - str(EVAL_RAC), + str(EVAL_ASDECIDED), "--out", str(records), "--answering-model", @@ -435,8 +448,8 @@ def test_stats_reproduces_the_hand_computed_mcnemar(tmp_path): completed = _run("run.py", "stats", "--records", str(records)) assert completed.returncode == 0, completed.stderr stats = json.loads(completed.stdout) - pair = stats["pairs"]["rac_vs_no_grounding"] - # fixture design: rac passes 3/3, no_grounding 1/3 -> a=1, b=2, c=0, d=0 + pair = stats["pairs"]["asdecided_vs_no_grounding"] + # fixture design: asdecided passes 3/3, no_grounding 1/3 -> a=1, b=2, c=0, d=0 assert pair["table"] == [1, 2, 0, 0] # exact two-sided binomial at min(2,0)=0 of 2 discordant: 2 * (1/4) = 0.5 assert pair["mcnemar"]["p_value"] == 0.5 @@ -446,17 +459,17 @@ def test_stats_reproduces_the_hand_computed_mcnemar(tmp_path): def test_stats_refuses_incomplete_required_arms(tmp_path): records = tmp_path / "resolution_records.jsonl" _score_records(tmp_path) - rac_ids = [ + asdecided_ids = [ json.loads(line)["example_id"] for line in records.read_text(encoding="utf-8").splitlines() - if json.loads(line)["arm"] == "rac" + if json.loads(line)["arm"] == "asdecided" ] - removed_id = rac_ids[-1] + removed_id = asdecided_ids[-1] kept = [ line for line in records.read_text(encoding="utf-8").splitlines() if not ( - json.loads(line)["arm"] == "rac" + json.loads(line)["arm"] == "asdecided" and json.loads(line)["example_id"] == removed_id ) ] @@ -467,7 +480,7 @@ def test_stats_refuses_incomplete_required_arms(tmp_path): "--records", str(records), "--require-arms", - "no_grounding,rac", + "no_grounding,asdecided", ) assert completed.returncode == 2 assert "incomplete paired records" in completed.stderr