diff --git a/gitchameleon/README.md b/gitchameleon/README.md index ad32692..071bf1b 100644 --- a/gitchameleon/README.md +++ b/gitchameleon/README.md @@ -30,9 +30,10 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, ## Design (GCB-ADR-0001) - **Arms** (DG-ADR-0001 single-variable pattern; held-constant answering - model): `no_grounding` / `rac` (live-decision retrieval over the example's - corpus via the shared harness runner — rac strictly as an external CLI) / - `naive_rag` (refuses until its embedder is pinned at funded-run time). + model): `no_grounding` / `asdecided` (live-decision retrieval over the + example's corpus via the shared harness runner — As Decided strictly as an + external CLI) / `naive_rag` (embedding retrieval over the identical corpus, pinned to + `voyage-4-large`, with query/document input types and cosine ranking). - **Corpus**: `build_corpus.py` turns each problem into a RAC decision artifact — the version pin, its rationale, companion pins, and the dataset's documentation links; never the solution, function name, or @@ -50,14 +51,14 @@ per-arm pass rate is SWE-DecisionBench's second co-primary outcome, ## Scaffold usage (offline, no model calls) ``` -python3 fetch_dataset.py # dataset/ (gitignored) + provenance pin +python3 fetch_dataset.py --revision 799a6a33e572a07a8985914e7251f5dea54b0ac4 python3 build_corpus.py # per-example corpora under corpus-build/ -python3 run.py --dry-run # per-example, per-arm prompt bundles +python3 run.py --dry-run # no_grounding + asdecided bundles python3 run.py --dry-run \ --dataset fixtures/sample_problems.json # the same, offline from the fixtures ``` -`rac` must be on `PATH` for the rac arm (external CLI only — no engine +`decided` must be on `PATH` for the `asdecided` arm (external CLI only — no engine imports, DG-ADR-0001). ## The funded run (GCB-ADR-0002 — the resolution co-primary pipeline) @@ -66,14 +67,25 @@ The pre-registered analysis and falsifier for this outcome live in `../decisiongrounding/spec/analysis-plan-amendment-1.md` (H2). Each step is resumable in isolation: -1. Pin the answering model (decisiongrounding pins `claude-opus-4-8`) and the - `naive_rag` embedder (`voyage:voyage-4-large` is the published strong - baseline there). -2. Answer every dry-run bundle: +1. Build all three arm bundles. The answering model is pinned to + `claude-opus-4-8`; the baseline embedder is pinned to + `voyage-4-large` (GCB-ADR-0003). Every frozen input is recorded in + [`run-config.json`](run-config.json): ``` + VOYAGE_API_KEY=... python3 run.py --dry-run \ + --arms no_grounding,asdecided,naive_rag --out out/bundles.jsonl + ``` + +2. Run a one-example-per-arm shakedown, inspect it, then answer every bundle. + `--resume` appends only missing example IDs, flushes every completion, and + records hashes of the exact prompt and grounding injected into the model: + + ``` + python3 run.py solutions --bundles out/bundles.jsonl \ + --answering claude --seed 0 --limit 1 --out out/shakedown python3 run.py solutions --bundles out/bundles.jsonl \ - --answering claude --seed 0 --out out/solutions + --answering claude --seed 0 --resume --out out/solutions ``` writes `out/solutions/solutions-.jsonl` in exactly the upstream @@ -81,21 +93,24 @@ resumable in isolation: ignored by upstream). `--answering offline-stub` exercises the plumbing keylessly; `litellm:` targets an OpenAI-compatible gateway. 3. Score each arm's file with the upstream harness — its executable tests - are the scorer; we add nothing. Clone GitChameleonBenchmark at a recorded - commit and run `evaluate --solution-path out/solutions/solutions-.jsonl` - (Docker; budget the per-version dependency installs). It writes + are the scorer; we add nothing. Clone GitChameleonBenchmark at commit + `3a1b6045a6b2a276bd24d715589cb041f8eccb93`, build its Docker image locally + from that checkout, and run + `evaluate --solution-path out/solutions/solutions-.jsonl` (budget the + per-version dependency installs). It writes `solutions-_eval_results.csv` next to each solution file. 4. Normalize the verdicts into paired resolution records (`schema/resolution_record.schema.json`) and run the pre-registered paired analysis: ``` - python3 run.py score --arm rac \ - --eval-results out/solutions/solutions-rac_eval_results.csv \ + python3 run.py score --arm asdecided \ + --eval-results out/solutions/solutions-asdecided_eval_results.csv \ --answering-model claude-opus-4-8 --upstream-harness \ --out out/resolution_records.jsonl python3 run.py score --arm no_grounding --eval-results … --append … - python3 run.py stats --records out/resolution_records.jsonl + python3 run.py stats --records out/resolution_records.jsonl \ + --require-arms no_grounding,asdecided,naive_rag ``` 5. Publish the records, per-arm pass rates, and stats with the dataset @@ -105,4 +120,5 @@ resumable in isolation: ## Local decisions -See [`decisions/`](decisions/) for benchmark-local design records. +See [`decisions/`](decisions/) for benchmark-local design records, including +the funded-run execution contract and pinning rationale (GCB-ADR-0003). diff --git a/gitchameleon/answer.py b/gitchameleon/answer.py index ea6736d..07d4721 100644 --- a/gitchameleon/answer.py +++ b/gitchameleon/answer.py @@ -21,6 +21,8 @@ import json import os +import time +import urllib.error import urllib.request # The held-constant instruction every arm shares; the task prompt itself is @@ -32,6 +34,7 @@ PINNED_CLAUDE = "claude-opus-4-8" MAX_TOKENS = 2048 +MAX_ATTEMPTS = 5 def _compose_user_prompt(prompt: str, grounding: list[str]) -> str: @@ -67,7 +70,9 @@ class ClaudeModel: version = PINNED_CLAUDE def __init__(self, seed: int = 0): - self.seed = seed # recorded for provenance; the pinned model rejects sampling seeds + self.seed = ( + seed # recorded for provenance; the pinned model rejects sampling seeds + ) if not os.environ.get("ANTHROPIC_API_KEY"): raise SystemExit("the claude answering model needs ANTHROPIC_API_KEY") self._client = None @@ -80,13 +85,36 @@ def _ensure_client(self): return self._client def complete(self, prompt: str, grounding: list[str]) -> str: - msg = self._ensure_client().messages.create( - model=self.version, - max_tokens=MAX_TOKENS, - system=SYSTEM, - messages=[{"role": "user", "content": _compose_user_prompt(prompt, grounding)}], + import anthropic + + retryable = ( + anthropic.RateLimitError, + anthropic.APIConnectionError, + anthropic.InternalServerError, ) - return "".join(b.text for b in msg.content if getattr(b, "type", "") == "text") + for attempt in range(MAX_ATTEMPTS): + try: + msg = self._ensure_client().messages.create( + model=self.version, + max_tokens=MAX_TOKENS, + system=SYSTEM, + messages=[ + { + "role": "user", + "content": _compose_user_prompt(prompt, grounding), + } + ], + ) + return "".join( + block.text + for block in msg.content + if getattr(block, "type", "") == "text" + ) + except retryable: + if attempt + 1 == MAX_ATTEMPTS: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") class LiteLLMModel: @@ -112,7 +140,10 @@ def complete(self, prompt: str, grounding: list[str]) -> str: "max_tokens": MAX_TOKENS, "messages": [ {"role": "system", "content": SYSTEM}, - {"role": "user", "content": _compose_user_prompt(prompt, grounding)}, + { + "role": "user", + "content": _compose_user_prompt(prompt, grounding), + }, ], } ).encode("utf-8") @@ -124,9 +155,23 @@ def complete(self, prompt: str, grounding: list[str]) -> str: "Authorization": f"Bearer {self._key}", }, ) - with urllib.request.urlopen(req, timeout=300) as resp: - payload = json.loads(resp.read().decode("utf-8")) - return payload["choices"][0]["message"]["content"] + for attempt in range(MAX_ATTEMPTS): + try: + with urllib.request.urlopen(req, timeout=300) as resp: + payload = json.loads(resp.read().decode("utf-8")) + return payload["choices"][0]["message"]["content"] + except urllib.error.HTTPError as exc: + retryable = exc.code == 429 or 500 <= exc.code < 600 + if not retryable or attempt + 1 == MAX_ATTEMPTS: + raise + retry_after = exc.headers.get("Retry-After") + delay = float(retry_after) if retry_after else min(2**attempt, 30) + time.sleep(delay) + except urllib.error.URLError: + if attempt + 1 == MAX_ATTEMPTS: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") def make_answering_model(name: str, seed: int = 0): diff --git a/gitchameleon/arms.py b/gitchameleon/arms.py index 98e6015..09b0417 100644 --- a/gitchameleon/arms.py +++ b/gitchameleon/arms.py @@ -5,8 +5,8 @@ answering model (the DG-ADR-0001 single-variable design): - ``no_grounding`` — the problem alone; the model answers from its weights. -- ``rac`` — the governing version-pin decision retrieved from the - example's corpus via the live-decision query (`rac find --decisions`), +- ``asdecided`` — the governing version-pin decision retrieved from the + example's corpus via the live-decision query (`decided find --decisions`), driven strictly as an external CLI through the shared harness runner. - ``naive_rag`` — embedding retrieval over the same corpus. Deliberately a seam: the embedder is pinned at funded-run time (mirroring @@ -16,17 +16,107 @@ from __future__ import annotations +import hashlib +import json +import math +import os +import time +import urllib.error +import urllib.request from pathlib import Path from harness.runner import RacRunner -ARMS = ("no_grounding", "rac", "naive_rag") -# How many retrieved decisions the rac arm feeds the answering model. -RAC_TOP_K = 3 +ARMS = ("no_grounding", "asdecided", "naive_rag") +# Both grounded arms supply the same number of artifacts to the answering model. +GROUNDING_TOP_K = 3 +NAIVE_RAG_MODEL = "voyage-4-large" +VOYAGE_EMBEDDINGS_URL = "https://api.voyageai.com/v1/embeddings" -def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: - """The rac arm: live-decision retrieval, retrieved artifacts verbatim. +class VoyageEmbedder: + """Pinned, retrieval-only baseline used by the funded naive_rag arm. + + Embeddings are cached by exact text within one run. This matters because + deterministic distractors recur across per-example corpora; repeated text + must not create repeated spend or a subtly different baseline. + """ + + model = NAIVE_RAG_MODEL + + def __init__(self, api_key: str | None = None, *, max_attempts: int = 5): + self.api_key = api_key or os.environ.get("VOYAGE_API_KEY") + if not self.api_key: + raise ValueError("the naive_rag arm needs VOYAGE_API_KEY") + self.max_attempts = max_attempts + self._cache: dict[tuple[str, str], list[float]] = {} + + def _request(self, texts: list[str], input_type: str) -> list[list[float]]: + body = json.dumps( + { + "input": texts, + "model": self.model, + "input_type": input_type, + "truncation": False, + } + ).encode("utf-8") + request = urllib.request.Request( + VOYAGE_EMBEDDINGS_URL, + data=body, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + }, + ) + for attempt in range(self.max_attempts): + try: + with urllib.request.urlopen(request, timeout=300) as response: + payload = json.loads(response.read().decode("utf-8")) + vectors = [entry["embedding"] for entry in payload["data"]] + if len(vectors) != len(texts): + raise RuntimeError( + f"Voyage returned {len(vectors)} vectors for {len(texts)} texts" + ) + return vectors + except urllib.error.HTTPError as exc: + retryable = exc.code == 429 or 500 <= exc.code < 600 + if not retryable or attempt + 1 == self.max_attempts: + raise + retry_after = exc.headers.get("Retry-After") + delay = float(retry_after) if retry_after else min(2**attempt, 30) + time.sleep(delay) + except urllib.error.URLError: + if attempt + 1 == self.max_attempts: + raise + time.sleep(min(2**attempt, 30)) + raise AssertionError("unreachable") + + def embed(self, texts: list[str], input_type: str) -> list[list[float]]: + keys = [ + (input_type, hashlib.sha256(text.encode("utf-8")).hexdigest()) + for text in texts + ] + missing: list[tuple[tuple[str, str], str]] = [ + (key, text) for key, text in zip(keys, texts) if key not in self._cache + ] + if missing: + vectors = self._request([text for _, text in missing], input_type) + for (key, _), vector in zip(missing, vectors): + self._cache[key] = vector + return [self._cache[key] for key in keys] + + +def _cosine(left: list[float], right: list[float]) -> float: + if len(left) != len(right): + raise ValueError("embedding dimensions differ") + numerator = sum(a * b for a, b in zip(left, right)) + left_norm = math.sqrt(sum(value * value for value in left)) + right_norm = math.sqrt(sum(value * value for value in right)) + return numerator / (left_norm * right_norm) if left_norm and right_norm else 0.0 + + +def asdecided_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: + """The As Decided arm: live-decision retrieval, artifacts verbatim. The query is what an agent grounding itself would ask before writing code against a library: which live decisions bind this dependency? @@ -34,7 +124,7 @@ def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: query = f"{row['library']} version pin" returned = runner.find_ids(query, str(corpus_dir), decisions=True) grounding: list[str] = [] - for artifact_id in returned[:RAC_TOP_K]: + for artifact_id in returned[:GROUNDING_TOP_K]: resolved = runner.resolve(artifact_id, str(corpus_dir)) payload = resolved.payload() if resolved.exit_code == 0 and "path" in payload: @@ -42,22 +132,45 @@ def rac_grounding(runner: RacRunner, corpus_dir: Path, row: dict) -> list[str]: return grounding +def naive_rag_grounding( + embedder: VoyageEmbedder, corpus_dir: Path, row: dict +) -> list[str]: + """Embedding retrieval over the exact corpus presented to RAC. + + Files are ranked by cosine similarity, then path for deterministic ties. + Query and document input types follow Voyage's retrieval contract. + """ + paths = sorted(corpus_dir.glob("*.md")) + documents = [path.read_text(encoding="utf-8") for path in paths] + if not documents: + return [] + document_vectors = embedder.embed(documents, "document") + query_vector = embedder.embed([f"{row['library']} version pin"], "query")[0] + ranked = sorted( + zip(paths, documents, document_vectors), + key=lambda item: (-_cosine(query_vector, item[2]), str(item[0])), + ) + return [document for _, document, _ in ranked[:GROUNDING_TOP_K]] + + def assemble_grounding( - arm: str, runner: RacRunner | None, corpus_dir: Path, row: dict + arm: str, + runner: RacRunner | None, + corpus_dir: Path, + row: dict, + embedder: VoyageEmbedder | None = None, ) -> list[str]: """The grounding context one arm supplies for one example.""" if arm == "no_grounding": return [] - if arm == "rac": + if arm == "asdecided": if runner is None: - raise ValueError("the rac arm needs a RacRunner (rac CLI on PATH)") - return rac_grounding(runner, corpus_dir, row) + raise ValueError("the asdecided arm needs the decided CLI on PATH") + return asdecided_grounding(runner, corpus_dir, row) if arm == "naive_rag": - raise NotImplementedError( - "naive_rag is a funded-run seam: pin the embedder there " - "(decisiongrounding pins voyage:voyage-4-large as the strong " - "published baseline) rather than shipping a weak lexical stand-in" - ) + if embedder is None: + raise ValueError("the naive_rag arm needs the pinned Voyage embedder") + return naive_rag_grounding(embedder, corpus_dir, row) raise ValueError(f"unknown arm {arm!r}; expected one of {ARMS}") diff --git a/gitchameleon/build_corpus.py b/gitchameleon/build_corpus.py index 3b12fc2..dad2048 100644 --- a/gitchameleon/build_corpus.py +++ b/gitchameleon/build_corpus.py @@ -6,7 +6,7 @@ respect: "this codebase targets ``==`` on Python ````". The builder writes that decision plus a deterministic set of distractor pins (other examples' decisions) into a per-example corpus -directory, so the rac arm's grounding retrieval has to find the governing pin +directory, so the As Decided arm's grounding retrieval has to find the governing pin among plausible competitors — the same distractor model decisiongrounding uses. @@ -39,7 +39,7 @@ def artifact_id(example_id: str) -> str: def decision_markdown(row: dict) -> str: - """One version-pin decision artifact, schema-valid for `rac validate`.""" + """One version-pin decision artifact, schema-valid for `decided validate`.""" library = row["library"] version = row["version"] python_version = row["python_version"] diff --git a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md index af2ed6a..7c5bd12 100644 --- a/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md +++ b/gitchameleon/decisions/GCB-ADR-0001-benchmark-scope.md @@ -37,17 +37,24 @@ prompt would measure prompt engineering, not grounding. The dataset is fetched on demand (stdlib script, provenance and content hash recorded), never vendored; three verbatim rows are committed as MIT-licensed -test fixtures with attribution. +test fixtures with attribution. The funded-run baseline is pinned by +GCB-ADR-0003 to Voyage `voyage-4-large`; it ranks the same Markdown artifacts +with explicit query/document embedding modes and deterministic cosine ties. ## Consequences -Grounding quality becomes the only manipulated variable, so a rac-vs- +The product-backed arm is named `asdecided` in CLI arguments, generated files, +records, and published results. The former `rac` label is not an accepted alias: +mixing both names would make paired-run completeness ambiguous and retain a +retired product name in durable evidence. + +Grounding quality becomes the only manipulated variable, so an asdecided-vs- no-grounding delta on the upstream pass rate is attributable to retrieval of the governing pin. The deliberate cost: our numbers are **not comparable to the upstream leaderboard**, which conditions the prompt on the version -explicitly — every published result must say so. The naive_rag arm refuses to -run until its embedder is pinned at funded-run time, so a weak stand-in can -never masquerade as the RAG baseline. +explicitly — every published result must say so. The naive_rag arm refuses +without the pinned embedder's API key, so a weak lexical stand-in can never +masquerade as the RAG baseline. ## Category diff --git a/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md b/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md new file mode 100644 index 0000000..f5e2b4c --- /dev/null +++ b/gitchameleon/decisions/GCB-ADR-0003-funded-run-execution.md @@ -0,0 +1,63 @@ +--- +schema_version: 1 +id: GCB-7M4VX2QK9D6H +type: decision +tags: [benchmark, execution, models, provenance] +--- +# GCB-ADR-0003: Pin and Checkpoint the Funded Evidence Run + +## Status + +Accepted + +## Context + +The GitChameleon evidence run makes 328 answering calls per arm and must +survive rate limits, transient provider failures, and operator interruption +without silently changing models or duplicating spend. GCB-ADR-0001 requires +a strong embedding baseline over the same corpus as As Decided, while +GCB-ADR-0002 requires paired, provenance-bearing resolution records. + +## Decision + +Pin the held-constant answering model to `claude-opus-4-8` and the naive RAG +embedder to `voyage-4-large`. Voyage receives the exact decision artifacts +with `input_type=document` and the shared ` version pin` query with +`input_type=query`; cosine similarity ranks the top three artifacts, with +path order breaking exact ties. Truncation is disabled so an over-limit input +fails rather than changing an arm invisibly. + +Answer generation is checkpointed per arm. Existing output is never +overwritten implicitly: a caller must select `--resume` or `--overwrite`. +Resume skips completed example IDs, flushes every new JSONL record, and stores +SHA-256 hashes of the exact task prompt and grounding list. Provider calls +retry only rate limits, connection failures, and server errors with bounded +backoff. A funded run begins with `--limit 1` for every arm before the full +call set is authorized. + +The upstream GitChameleon commit remains a separate, mandatory scoring pin. +Neither an answering completion nor a locally normalized record is a pass; +only the upstream executable harness supplies that verdict. + +The concrete frozen values live in `../run-config.json`. The first registered +run uses dataset commit `799a6a33e572a07a8985914e7251f5dea54b0ac4` +(328 raw JSONL rows) and upstream harness commit +`3a1b6045a6b2a276bd24d715589cb041f8eccb93`. The scorer image is built locally +from that checkout; the upstream floating `latest` image is not evidence. + +## Consequences + +Interrupted runs can continue without duplicate calls, the three arms remain +auditable at the injection boundary, and the naive RAG label names a concrete +strong baseline. The evidence run still requires two owner-supplied provider +credentials and an upstream scoring environment. No result exists until all +three arm files are scored and the paired records pass completeness checks. + +## Category + +Process + +## Related Decisions + +- GCB-329CD3DAMG8Y +- GCB-KWRRD0T8K2Z9 diff --git a/gitchameleon/fetch_dataset.py b/gitchameleon/fetch_dataset.py index 39e0d24..e0e33e7 100644 --- a/gitchameleon/fetch_dataset.py +++ b/gitchameleon/fetch_dataset.py @@ -4,8 +4,8 @@ Downloads all rows of the upstream dataset — `cabbage972/GitChameleon-2.0` (MIT), the dataset behind GitChameleon 2.0 (arXiv:2507.12367; upstream code -Apache-2.0 at github.com/mrcabbage972/GitChameleonBenchmark) — via the -HuggingFace datasets-server API, stdlib only. Writes: +Apache-2.0 at github.com/mrcabbage972/GitChameleonBenchmark) — from the raw +JSONL at an exact Hugging Face commit, stdlib only. Writes: - ``dataset/problems.json`` — the rows, exactly as served. - ``dataset/provenance.json`` — the dataset revision, row count, retrieval @@ -18,18 +18,18 @@ from __future__ import annotations +import argparse import hashlib import json import sys import urllib.request from datetime import UTC, datetime from pathlib import Path +from urllib.parse import quote DATASET = "cabbage972/GitChameleon-2.0" CONFIG = "problems" SPLIT = "train" -PAGE = 100 -BASE = "https://datasets-server.huggingface.co" OUT_DIR = Path(__file__).resolve().parent / "dataset" @@ -38,36 +38,51 @@ def _get_json(url: str) -> dict: return json.load(response) -def fetch_rows() -> list[dict]: - rows: list[dict] = [] - offset = 0 - while True: - payload = _get_json( - f"{BASE}/rows?dataset={DATASET}&config={CONFIG}&split={SPLIT}" - f"&offset={offset}&length={PAGE}" - ) - batch = [entry["row"] for entry in payload["rows"]] - if not batch: - break - rows.extend(batch) - offset += len(batch) - if len(batch) < PAGE: - break - return rows +def fetch_rows(revision: str) -> list[dict]: + """Fetch the immutable raw JSONL at one exact Hub commit.""" + dataset_path = quote(DATASET, safe="/") + revision_path = quote(revision, safe="") + url = ( + f"https://huggingface.co/datasets/{dataset_path}/resolve/" + f"{revision_path}/dataset.jsonl" + ) + with urllib.request.urlopen(url) as response: + return [ + json.loads(line) + for line in response.read().decode("utf-8").splitlines() + if line.strip() + ] def dataset_revision() -> str: - payload = _get_json(f"https://huggingface.co/api/datasets/{DATASET.replace('/', '%2F')}") + # Hugging Face's repository endpoint treats owner/name as path segments; + # percent-encoding the slash returns HTTP 400. + payload = _get_json( + f"https://huggingface.co/api/datasets/{quote(DATASET, safe='/')}" + ) return str(payload.get("sha", "unknown")) def rows_hash(rows: list[dict]) -> str: - canonical = json.dumps(rows, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + canonical = json.dumps( + rows, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() -def main() -> int: - rows = fetch_rows() +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--revision", + default=None, + help="exact Hugging Face dataset commit (default: resolve current HEAD once)", + ) + args = parser.parse_args(argv) + revision = args.revision or dataset_revision() + if revision == "unknown": + print("fetch_dataset: upstream returned no dataset revision", file=sys.stderr) + return 1 + rows = fetch_rows(revision) if not rows: print("fetch_dataset: upstream returned no rows", file=sys.stderr) return 1 @@ -80,7 +95,7 @@ def main() -> int: "config": CONFIG, "split": SPLIT, "license": "MIT", - "revision": dataset_revision(), + "revision": revision, "n_rows": len(rows), "rows_hash": rows_hash(rows), "fetched_at": datetime.now(UTC).isoformat(), diff --git a/gitchameleon/fixtures/sample_eval_results_rac.csv b/gitchameleon/fixtures/sample_eval_results_asdecided.csv similarity index 100% rename from gitchameleon/fixtures/sample_eval_results_rac.csv rename to gitchameleon/fixtures/sample_eval_results_asdecided.csv diff --git a/gitchameleon/run-config.json b/gitchameleon/run-config.json new file mode 100644 index 0000000..390eadb --- /dev/null +++ b/gitchameleon/run-config.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "preregistered-not-run", + "arms": [ + "no_grounding", + "asdecided", + "naive_rag" + ], + "seed": 0, + "dataset": { + "id": "cabbage972/GitChameleon-2.0", + "revision": "799a6a33e572a07a8985914e7251f5dea54b0ac4", + "rows": 328, + "rows_hash": "sha256:5fb3b2be20f180be0842950ad96d5c19a92b9cf96c45eb2896addc8c0479ece3", + "source_file": "dataset.jsonl" + }, + "answering": { + "provider": "anthropic", + "model": "claude-opus-4-8", + "max_tokens": 2048 + }, + "naive_rag": { + "provider": "voyage", + "model": "voyage-4-large", + "top_k": 3, + "similarity": "cosine", + "document_input_type": "document", + "query_input_type": "query", + "truncation": false + }, + "asdecided": { + "executable": "decided", + "minimum_version": "0.26.2", + "top_k": 3 + }, + "upstream_harness": { + "repository": "https://github.com/mrcabbage972/GitChameleonBenchmark", + "commit": "3a1b6045a6b2a276bd24d715589cb041f8eccb93", + "execution": "build-local-docker-image-from-pinned-commit" + }, + "protocol": { + "prompt_restates_version": false, + "upstream_leaderboard_comparable": false, + "merge_gate": false + } +} diff --git a/gitchameleon/run.py b/gitchameleon/run.py index 8207322..c5ed59a 100644 --- a/gitchameleon/run.py +++ b/gitchameleon/run.py @@ -31,6 +31,7 @@ import argparse import csv +import hashlib import json import sys from pathlib import Path @@ -38,9 +39,9 @@ BENCHMARK_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(BENCHMARK_DIR.parent)) -from harness.runner import RacRunner # noqa: E402 +import arms as arms_mod -import arms as arms_mod # noqa: E402 +from harness.runner import RacRunner EXIT_OK = 0 EXIT_USAGE = 2 @@ -57,8 +58,22 @@ def _dataset_revision(provenance_path: Path) -> str | None: return None -def dry_run(rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: Path) -> int: - runner = RacRunner() if "rac" in arm_names else None +def _prompt_hash(bundle: dict) -> str: + return hashlib.sha256(bundle["prompt"].encode("utf-8")).hexdigest() + + +def _grounding_hash(bundle: dict) -> str: + grounding = bundle.get("grounding") or [] + return hashlib.sha256( + json.dumps(grounding, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + ).hexdigest() + + +def dry_run( + rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: Path +) -> int: + runner = RacRunner() if "asdecided" in arm_names else None + embedder = arms_mod.VoyageEmbedder() if "naive_rag" in arm_names else None bundles = 0 with out_path.open("w", encoding="utf-8") as out: for row in rows: @@ -71,7 +86,9 @@ def dry_run(rows: list[dict], corpus_root: Path, arm_names: list[str], out_path: ) return EXIT_USAGE for arm in arm_names: - grounding = arms_mod.assemble_grounding(arm, runner, corpus_dir, row) + grounding = arms_mod.assemble_grounding( + arm, runner, corpus_dir, row, embedder=embedder + ) out.write( json.dumps( { @@ -97,49 +114,162 @@ def cmd_solutions(args) -> int: bundles_path = Path(args.bundles) if not bundles_path.is_file(): - print(f"gitchameleon: bundles not found: {bundles_path} — run --dry-run first", - file=sys.stderr) + print( + f"gitchameleon: bundles not found: {bundles_path} — run --dry-run first", + file=sys.stderr, + ) + return EXIT_USAGE + if args.resume and args.overwrite: + print( + "gitchameleon: --resume and --overwrite are mutually exclusive", + file=sys.stderr, + ) + return EXIT_USAGE + requested_arms = [arm.strip() for arm in args.arms.split(",") if arm.strip()] + unknown = [arm for arm in requested_arms if arm not in arms_mod.ARMS] + if unknown: + print( + f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", + file=sys.stderr, + ) + return EXIT_USAGE + + bundles = [ + json.loads(line) + for line in bundles_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + bundles = [bundle for bundle in bundles if bundle["arm"] in requested_arms] + if args.limit is not None: + per_arm: dict[str, int] = {} + limited: list[dict] = [] + for bundle in bundles: + arm = bundle["arm"] + if per_arm.get(arm, 0) >= args.limit: + continue + limited.append(bundle) + per_arm[arm] = per_arm.get(arm, 0) + 1 + bundles = limited + if not bundles: + print("gitchameleon: no bundles selected", file=sys.stderr) return EXIT_USAGE + model = make_answering_model(args.answering, args.seed) revision = _dataset_revision(Path(args.provenance)) if revision is None: - print(f"gitchameleon: warning — no dataset provenance at {args.provenance}; " - "the records will carry dataset_revision=null", file=sys.stderr) + print( + f"gitchameleon: warning — no dataset provenance at {args.provenance}; " + "the records will carry dataset_revision=null", + file=sys.stderr, + ) out_dir = Path(args.out) out_dir.mkdir(parents=True, exist_ok=True) - handles: dict[str, object] = {} - counts: dict[str, int] = {} - try: - with bundles_path.open(encoding="utf-8") as fp: - for line in fp: + selected_arms = sorted({bundle["arm"] for bundle in bundles}) + bundle_by_key = { + (bundle["arm"], str(bundle["example_id"])): bundle for bundle in bundles + } + completed: dict[str, set[str]] = {arm: set() for arm in selected_arms} + for arm in selected_arms: + path = out_dir / f"solutions-{arm}.jsonl" + if not path.exists(): + continue + if not args.resume and not args.overwrite: + print( + f"gitchameleon: refusing to overwrite {path}; pass --resume or --overwrite", + file=sys.stderr, + ) + return EXIT_USAGE + if args.resume: + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), 1 + ): if not line.strip(): continue - bundle = json.loads(line) - arm = bundle["arm"] - answer = model.complete(bundle["prompt"], bundle.get("grounding") or []) - if arm not in handles: - handles[arm] = (out_dir / f"solutions-{arm}.jsonl").open("w", encoding="utf-8") - # Upstream Solution model reads example_id + answer and ignores - # the extra provenance fields. - handles[arm].write(json.dumps({ - "example_id": str(bundle["example_id"]), - "answer": answer, - "arm": arm, - "library": bundle.get("library"), - "version": bundle.get("version"), + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + print( + f"gitchameleon: invalid resume record {path}:{line_number}: {exc}", + file=sys.stderr, + ) + return EXIT_USAGE + record_key = (arm, str(record["example_id"])) + bundle = bundle_by_key.get(record_key) + expected = { "answering_model": model.version, "seed": args.seed, "dataset_revision": revision, - }, ensure_ascii=False) + "\n") - counts[arm] = counts.get(arm, 0) + 1 + "prompt_sha256": _prompt_hash(bundle) if bundle else None, + "grounding_sha256": _grounding_hash(bundle) if bundle else None, + } + mismatched = { + field: {"found": record.get(field), "expected": value} + for field, value in expected.items() + if record.get(field) != value + } + if mismatched: + print( + f"gitchameleon: cannot resume incompatible record " + f"{path}:{line_number}: {mismatched}", + file=sys.stderr, + ) + return EXIT_USAGE + completed[arm].add(str(record["example_id"])) + + handles: dict[str, object] = {} + counts: dict[str, int] = {} + skipped: dict[str, int] = {} + try: + for bundle in bundles: + arm = bundle["arm"] + example_id = str(bundle["example_id"]) + if example_id in completed[arm]: + skipped[arm] = skipped.get(arm, 0) + 1 + continue + grounding = bundle.get("grounding") or [] + answer = model.complete(bundle["prompt"], grounding) + if arm not in handles: + mode = "a" if args.resume else "w" + handles[arm] = (out_dir / f"solutions-{arm}.jsonl").open( + mode, encoding="utf-8" + ) + # Upstream Solution reads example_id + answer and ignores extras. + handles[arm].write( + json.dumps( + { + "example_id": example_id, + "answer": answer, + "arm": arm, + "library": bundle.get("library"), + "version": bundle.get("version"), + "answering_model": model.version, + "seed": args.seed, + "dataset_revision": revision, + "prompt_sha256": _prompt_hash(bundle), + "grounding_sha256": _grounding_hash(bundle), + }, + ensure_ascii=False, + ) + + "\n" + ) + handles[arm].flush() + counts[arm] = counts.get(arm, 0) + 1 + print(f"answered {arm} example {example_id}", file=sys.stderr) finally: for h in handles.values(): h.close() - for arm, n in sorted(counts.items()): - print(f"wrote {n} solutions -> {out_dir / f'solutions-{arm}.jsonl'}") - print("score each file with the upstream harness: " - "evaluate --solution-path (GitChameleonBenchmark)") + for arm in selected_arms: + n = counts.get(arm, 0) + already = skipped.get(arm, 0) + print( + f"wrote {n} solutions ({already} resumed) -> " + f"{out_dir / f'solutions-{arm}.jsonl'}" + ) + print( + "score each file with the upstream harness: " + "evaluate --solution-path (GitChameleonBenchmark)" + ) return EXIT_OK @@ -162,14 +292,19 @@ def cmd_score(args) -> int: "example_id": eid, "arm": args.arm, "passed": _truthy(row.get("passed", "")), - "compiled": _truthy(row["compiled"]) if row.get("compiled") not in (None, "") else None, + "compiled": _truthy(row["compiled"]) + if row.get("compiled") not in (None, "") + else None, "answering_model": args.answering_model, "seed": args.seed, - "dataset_revision": args.dataset_revision or _dataset_revision(Path(args.provenance)), + "dataset_revision": args.dataset_revision + or _dataset_revision(Path(args.provenance)), "upstream_harness": args.upstream_harness, } if not by_example: - print("gitchameleon: no solution_code rows found in the eval CSV", file=sys.stderr) + print( + "gitchameleon: no solution_code rows found in the eval CSV", file=sys.stderr + ) return EXIT_USAGE out_path = Path(args.out) out_path.parent.mkdir(parents=True, exist_ok=True) @@ -178,8 +313,10 @@ def cmd_score(args) -> int: for eid in sorted(by_example, key=int): out.write(json.dumps(by_example[eid], ensure_ascii=False) + "\n") passed = sum(1 for r in by_example.values() if r["passed"]) - print(f"{args.arm}: {passed}/{len(by_example)} passed -> {out_path}" - f" ({'appended' if args.append else 'wrote'})") + print( + f"{args.arm}: {passed}/{len(by_example)} passed -> {out_path}" + f" ({'appended' if args.append else 'wrote'})" + ) return EXIT_OK @@ -190,10 +327,43 @@ def cmd_stats(args) -> int: print(f"gitchameleon: records not found: {records_path}", file=sys.stderr) return EXIT_USAGE sys.path.insert(0, str(BENCHMARK_DIR.parent / "decisiongrounding")) - from scoring.stats import paired_significance # noqa: E402 - - records = [json.loads(line) for line in records_path.read_text(encoding="utf-8").splitlines() - if line.strip()] + from scoring.stats import paired_significance + + records = [ + json.loads(line) + for line in records_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + required_arms = [arm.strip() for arm in args.require_arms.split(",") if arm.strip()] + by_arm: dict[str, set[str]] = {} + seen: set[tuple[str, str, object]] = set() + for record in records: + key = (record["arm"], str(record["example_id"]), record.get("seed")) + if key in seen: + print(f"gitchameleon: duplicate resolution record {key}", file=sys.stderr) + return EXIT_USAGE + seen.add(key) + by_arm.setdefault(record["arm"], set()).add(str(record["example_id"])) + missing_arms = [arm for arm in required_arms if arm not in by_arm] + if missing_arms: + print( + f"gitchameleon: required arm(s) missing from records: {missing_arms}", + file=sys.stderr, + ) + return EXIT_USAGE + compared = required_arms or sorted(by_arm) + if len(compared) < 2: + print("gitchameleon: stats needs at least two arms", file=sys.stderr) + return EXIT_USAGE + expected = by_arm[compared[0]] + incomplete = { + arm: sorted(expected.symmetric_difference(by_arm[arm]), key=int) + for arm in compared[1:] + if by_arm[arm] != expected + } + if incomplete: + print(f"gitchameleon: incomplete paired records: {incomplete}", file=sys.stderr) + return EXIT_USAGE out = paired_significance(records, outcome="passed") rendered = json.dumps(out, indent=2) if args.out: @@ -206,60 +376,131 @@ def cmd_stats(args) -> int: def _add_common_provenance(sub) -> None: - sub.add_argument("--provenance", default=str(BENCHMARK_DIR / "dataset" / "provenance.json"), - help="fetch_dataset.py provenance pin (for dataset_revision).") + sub.add_argument( + "--provenance", + default=str(BENCHMARK_DIR / "dataset" / "provenance.json"), + help="fetch_dataset.py provenance pin (for dataset_revision).", + ) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( prog="gitchameleon", description="GitChameleon evidence run: dry-run offline; " - "solutions/score/stats are the funded-run pipeline.", + "solutions/score/stats are the funded-run pipeline.", ) sub = parser.add_subparsers(dest="mode") - p_sol = sub.add_parser("solutions", help="answer each bundle; write per-arm solution JSONL") - p_sol.add_argument("--bundles", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl")) - p_sol.add_argument("--answering", default="offline-stub", - help="offline-stub | claude | litellm:") + p_sol = sub.add_parser( + "solutions", help="answer each bundle; write per-arm solution JSONL" + ) + p_sol.add_argument( + "--bundles", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl") + ) + p_sol.add_argument( + "--answering", + default="offline-stub", + help="offline-stub | claude | litellm:", + ) + p_sol.add_argument( + "--arms", + default=",".join(arms_mod.ARMS), + help="comma-separated subset of arms to answer", + ) + p_sol.add_argument( + "--limit", + type=int, + default=None, + help="answer at most N examples per selected arm (shakedowns only)", + ) + p_sol.add_argument( + "--resume", + action="store_true", + help="append only missing example ids to existing arm files", + ) + p_sol.add_argument( + "--overwrite", + action="store_true", + help="replace existing selected-arm solution files", + ) p_sol.add_argument("--seed", type=int, default=0) p_sol.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "solutions")) _add_common_provenance(p_sol) p_sol.set_defaults(func=cmd_solutions) - p_score = sub.add_parser("score", help="normalize an upstream eval_results CSV " - "into paired resolution records") + p_score = sub.add_parser( + "score", + help="normalize an upstream eval_results CSV into paired resolution records", + ) p_score.add_argument("--arm", required=True, choices=arms_mod.ARMS) - p_score.add_argument("--eval-results", required=True, - help="the upstream harness's *_eval_results.csv for this arm") - p_score.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl")) - p_score.add_argument("--append", action="store_true", - help="append to --out (one file across arms)") - p_score.add_argument("--answering-model", default=None, - help="the pinned answering model the solutions used") + p_score.add_argument( + "--eval-results", + required=True, + help="the upstream harness's *_eval_results.csv for this arm", + ) + p_score.add_argument( + "--out", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl") + ) + p_score.add_argument( + "--append", action="store_true", help="append to --out (one file across arms)" + ) + p_score.add_argument( + "--answering-model", + default=None, + help="the pinned answering model the solutions used", + ) p_score.add_argument("--seed", type=int, default=0) p_score.add_argument("--dataset-revision", default=None) - p_score.add_argument("--upstream-harness", default=None, - help="GitChameleonBenchmark commit/tag used to score") + p_score.add_argument( + "--upstream-harness", + default=None, + help="GitChameleonBenchmark commit/tag used to score", + ) _add_common_provenance(p_score) p_score.set_defaults(func=cmd_score) - p_stats = sub.add_parser("stats", help="paired significance over resolution records") - p_stats.add_argument("--records", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl")) - p_stats.add_argument("--out", default=None, help="write JSON here instead of stdout") + p_stats = sub.add_parser( + "stats", help="paired significance over resolution records" + ) + p_stats.add_argument( + "--records", default=str(BENCHMARK_DIR / "out" / "resolution_records.jsonl") + ) + p_stats.add_argument( + "--out", default=None, help="write JSON here instead of stdout" + ) + p_stats.add_argument( + "--require-arms", + default="", + help="comma-separated arms that must all contain the same example IDs", + ) p_stats.set_defaults(func=cmd_stats) # Legacy top-level dry-run flags (the offline scaffold surface). - parser.add_argument("--dry-run", action="store_true", - help="Assemble per-example, per-arm prompt bundles without any model call.") - parser.add_argument("--dataset", default=str(BENCHMARK_DIR / "dataset" / "problems.json"), - help="Problem rows (fetch_dataset.py output, or the committed fixture).") - parser.add_argument("--corpus", default=str(BENCHMARK_DIR / "corpus-build"), - help="Per-example corpus root (build_corpus.py output).") - parser.add_argument("--arms", default="no_grounding,rac", - help="Comma-separated arms (naive_rag refuses until its embedder is pinned).") - parser.add_argument("--out", default=str(BENCHMARK_DIR / "out" / "bundles.jsonl"), - help="Where the dry-run writes the prompt bundles (JSONL).") + parser.add_argument( + "--dry-run", + action="store_true", + help="Assemble per-example, per-arm prompt bundles without any model call.", + ) + parser.add_argument( + "--dataset", + default=str(BENCHMARK_DIR / "dataset" / "problems.json"), + help="Problem rows (fetch_dataset.py output, or the committed fixture).", + ) + parser.add_argument( + "--corpus", + default=str(BENCHMARK_DIR / "corpus-build"), + help="Per-example corpus root (build_corpus.py output).", + ) + parser.add_argument( + "--arms", + default="no_grounding,asdecided", + help="Comma-separated arms (naive_rag refuses until its embedder is pinned).", + ) + parser.add_argument( + "--out", + default=str(BENCHMARK_DIR / "out" / "bundles.jsonl"), + help="Where the dry-run writes the prompt bundles (JSONL).", + ) args = parser.parse_args(argv) if getattr(args, "mode", None): @@ -268,7 +509,10 @@ def main(argv: list[str] | None = None) -> int: arm_names = [arm.strip() for arm in args.arms.split(",") if arm.strip()] unknown = [arm for arm in arm_names if arm not in arms_mod.ARMS] if unknown: - print(f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", file=sys.stderr) + print( + f"gitchameleon: unknown arm(s) {unknown}; expected {arms_mod.ARMS}", + file=sys.stderr, + ) return EXIT_USAGE dataset_path = Path(args.dataset) @@ -293,7 +537,7 @@ def main(argv: list[str] | None = None) -> int: out_path.parent.mkdir(parents=True, exist_ok=True) try: return dry_run(load_rows(dataset_path), Path(args.corpus), arm_names, out_path) - except NotImplementedError as exc: + except (NotImplementedError, ValueError) as exc: print(f"gitchameleon: {exc}", file=sys.stderr) return EXIT_USAGE diff --git a/gitchameleon/schema/resolution_record.schema.json b/gitchameleon/schema/resolution_record.schema.json index da90e92..5fea8cb 100644 --- a/gitchameleon/schema/resolution_record.schema.json +++ b/gitchameleon/schema/resolution_record.schema.json @@ -13,7 +13,7 @@ }, "arm": { "type": "string", - "enum": ["no_grounding", "rac", "naive_rag"] + "enum": ["no_grounding", "asdecided", "naive_rag"] }, "passed": { "type": "boolean", diff --git a/tests/test_gitchameleon_scaffold.py b/tests/test_gitchameleon_scaffold.py index 97b879d..ad7e93c 100644 --- a/tests/test_gitchameleon_scaffold.py +++ b/tests/test_gitchameleon_scaffold.py @@ -2,8 +2,8 @@ """GitChameleon evidence-run scaffold battery (offline; fixtures only). Exercises the key-less surface: the corpus builder is deterministic and emits -schema-valid RAC decisions, the dry-run assembles honest prompt bundles (the -rac arm's grounding leads with the governing pin; prompts never leak the +schema-valid As Decided decisions, the dry-run assembles honest prompt bundles +(the As Decided arm's grounding leads with the governing pin; prompts never leak the pinned version; grounding never leaks solutions or tests), and the funded-run seams refuse loudly instead of running weak stand-ins. """ @@ -18,6 +18,9 @@ GCB = REPO_ROOT / "gitchameleon" FIXTURES = GCB / "fixtures" / "sample_problems.json" +sys.path.insert(0, str(GCB)) +import arms as arms_mod +import fetch_dataset as fetch_mod def _run(script: str, *args: str) -> subprocess.CompletedProcess: @@ -32,7 +35,13 @@ def _run(script: str, *args: str) -> subprocess.CompletedProcess: def _build(tmp_path, name: str = "corpus"): out = tmp_path / name completed = _run( - "build_corpus.py", "--dataset", str(FIXTURES), "--out", str(out), "--distractors", "2" + "build_corpus.py", + "--dataset", + str(FIXTURES), + "--out", + str(out), + "--distractors", + "2", ) assert completed.returncode == 0, completed.stderr return out @@ -40,7 +49,8 @@ def _build(tmp_path, name: str = "corpus"): def _corpus_bytes(root) -> dict[str, bytes]: return { - str(path.relative_to(root)): path.read_bytes() for path in sorted(root.rglob("*.md")) + str(path.relative_to(root)): path.read_bytes() + for path in sorted(root.rglob("*.md")) } @@ -48,16 +58,69 @@ def _fixture_rows() -> list[dict]: return json.loads(FIXTURES.read_text(encoding="utf-8"))["rows"] +def test_funded_run_config_matches_implemented_pins(): + config = json.loads((GCB / "run-config.json").read_text(encoding="utf-8")) + assert config["status"] == "preregistered-not-run" + assert config["arms"] == list(arms_mod.ARMS) + assert config["answering"]["model"] == "claude-opus-4-8" + assert config["naive_rag"]["model"] == arms_mod.NAIVE_RAG_MODEL + assert config["naive_rag"]["top_k"] == arms_mod.GROUNDING_TOP_K + assert len(config["dataset"]["revision"]) == 40 + assert len(config["upstream_harness"]["commit"]) == 40 + + def test_corpus_builder_is_deterministic(tmp_path): first = _corpus_bytes(_build(tmp_path, "a")) second = _corpus_bytes(_build(tmp_path, "b")) assert first == second +def test_dataset_revision_preserves_the_owner_name_path(monkeypatch): + seen = [] + + def fake_get(url): + seen.append(url) + return {"sha": "dataset-pin"} + + monkeypatch.setattr(fetch_mod, "_get_json", fake_get) + assert fetch_mod.dataset_revision() == "dataset-pin" + assert seen == ["https://huggingface.co/api/datasets/cabbage972/GitChameleon-2.0"] + + +def test_dataset_rows_are_fetched_at_the_exact_revision(monkeypatch): + seen = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *_): + return None + + def read(self): + return b'{"example_id":"7"}\n' + + def fake_open(url): + seen.append(url) + return Response() + + monkeypatch.setattr(fetch_mod.urllib.request, "urlopen", fake_open) + assert fetch_mod.fetch_rows("abc123") == [{"example_id": "7"}] + assert seen == [ + ( + "https://huggingface.co/datasets/cabbage972/GitChameleon-2.0/" + "resolve/abc123/dataset.jsonl" + ) + ] + + def test_built_corpora_are_schema_valid(tmp_path): corpus = _build(tmp_path) completed = subprocess.run( - ["decided", "validate", str(corpus)], capture_output=True, text=True, check=False + ["decided", "validate", str(corpus)], + capture_output=True, + text=True, + check=False, ) assert completed.returncode == 0, completed.stdout + completed.stderr @@ -66,12 +129,14 @@ def test_decision_artifacts_never_leak_solutions_or_tests(tmp_path): corpus = _build(tmp_path) leaks = [row["solution"].strip() for row in _fixture_rows()] leaks += [row["test"].strip() for row in _fixture_rows()] - corpus_text = "\n".join(text.decode("utf-8") for text in _corpus_bytes(corpus).values()) + corpus_text = "\n".join( + text.decode("utf-8") for text in _corpus_bytes(corpus).values() + ) for leak in leaks: assert leak not in corpus_text -def _dry_run_bundles(tmp_path, arms: str = "no_grounding,rac") -> list[dict]: +def _dry_run_bundles(tmp_path, arms: str = "no_grounding,asdecided") -> list[dict]: corpus = _build(tmp_path) out = tmp_path / "bundles.jsonl" completed = _run( @@ -90,11 +155,11 @@ def _dry_run_bundles(tmp_path, arms: str = "no_grounding,rac") -> list[dict]: return [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] -def test_rac_arm_grounding_leads_with_the_governing_pin(tmp_path): +def test_asdecided_arm_grounding_leads_with_the_governing_pin(tmp_path): bundles = _dry_run_bundles(tmp_path) - rac_bundles = [b for b in bundles if b["arm"] == "rac"] - assert len(rac_bundles) == len(_fixture_rows()) - for bundle in rac_bundles: + asdecided_bundles = [b for b in bundles if b["arm"] == "asdecided"] + assert len(asdecided_bundles) == len(_fixture_rows()) + for bundle in asdecided_bundles: assert bundle["grounding"], bundle["example_id"] heading = f"# Library Version Pin: {bundle['library']} {bundle['version']}" assert heading in bundle["grounding"][0] @@ -113,7 +178,8 @@ def test_prompt_never_states_the_pinned_version(tmp_path): assert bundle["version"] not in bundle["prompt"] -def test_naive_rag_refuses_until_embedder_is_pinned(tmp_path): +def test_naive_rag_refuses_without_the_pinned_embedder_key(tmp_path, monkeypatch): + monkeypatch.delenv("VOYAGE_API_KEY", raising=False) corpus = _build(tmp_path) completed = _run( "run.py", @@ -128,7 +194,24 @@ def test_naive_rag_refuses_until_embedder_is_pinned(tmp_path): str(tmp_path / "bundles.jsonl"), ) assert completed.returncode == 2 - assert "embedder" in completed.stderr + assert "VOYAGE_API_KEY" in completed.stderr + + +def test_naive_rag_ranks_the_governing_pin_first(tmp_path): + corpus = _build(tmp_path) + row = _fixture_rows()[0] + corpus_dir = corpus / f"example-{row['example_id']}" + + class FakeEmbedder: + def embed(self, texts, input_type): + if input_type == "query": + return [[1.0, 0.0]] + heading = f"# Library Version Pin: {row['library']} {row['version']}" + return [[1.0, 0.0] if heading in text else [0.0, 1.0] for text in texts] + + grounding = arms_mod.naive_rag_grounding(FakeEmbedder(), corpus_dir, row) + assert len(grounding) == 3 + assert f"# Library Version Pin: {row['library']} {row['version']}" in grounding[0] def test_bare_invocation_points_at_the_modes(tmp_path): @@ -137,9 +220,22 @@ def test_bare_invocation_points_at_the_modes(tmp_path): assert "solutions / score / stats" in completed.stderr +def test_retired_rac_arm_name_is_rejected(): + completed = _run( + "run.py", + "--dry-run", + "--dataset", + str(FIXTURES), + "--arms", + "rac", + ) + assert completed.returncode == 2 + assert "unknown arm(s) ['rac']" in completed.stderr + + # --- the resolution co-primary pipeline (GCB-ADR-0002), offline --------------- -EVAL_RAC = GCB / "fixtures" / "sample_eval_results_rac.csv" +EVAL_ASDECIDED = GCB / "fixtures" / "sample_eval_results_asdecided.csv" EVAL_NONE = GCB / "fixtures" / "sample_eval_results_no_grounding.csv" @@ -147,8 +243,14 @@ def _bundles_file(tmp_path) -> str: corpus = _build(tmp_path) out = tmp_path / "bundles.jsonl" completed = _run( - "run.py", "--dry-run", "--dataset", str(FIXTURES), "--corpus", str(corpus), - "--out", str(out), + "run.py", + "--dry-run", + "--dataset", + str(FIXTURES), + "--corpus", + str(corpus), + "--out", + str(out), ) assert completed.returncode == 0, completed.stderr return str(out) @@ -156,14 +258,22 @@ def _bundles_file(tmp_path) -> str: def _solutions(tmp_path, out_name: str) -> dict[str, list[dict]]: completed = _run( - "run.py", "solutions", "--bundles", _bundles_file(tmp_path), - "--answering", "offline-stub", "--out", str(tmp_path / out_name), + "run.py", + "solutions", + "--bundles", + _bundles_file(tmp_path), + "--answering", + "offline-stub", + "--out", + str(tmp_path / out_name), ) assert completed.returncode == 0, completed.stderr out: dict[str, list[dict]] = {} for path in sorted((tmp_path / out_name).glob("solutions-*.jsonl")): arm = path.stem.removeprefix("solutions-") - out[arm] = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + out[arm] = [ + json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + ] return out @@ -171,7 +281,7 @@ def test_offline_stub_solutions_are_deterministic_and_upstream_shaped(tmp_path): first = _solutions(tmp_path, "sol-a") second = _solutions(tmp_path, "sol-b") assert first == second - assert set(first) == {"no_grounding", "rac"} + assert set(first) == {"no_grounding", "asdecided"} for arm, records in first.items(): assert len(records) == len(_fixture_rows()) for rec in records: @@ -179,13 +289,106 @@ def test_offline_stub_solutions_are_deterministic_and_upstream_shaped(tmp_path): assert isinstance(rec["example_id"], str) and rec["answer"] assert rec["arm"] == arm assert "offline-stub" in rec["answer"] # plumbing output is labelled + assert len(rec["prompt_sha256"]) == 64 + assert len(rec["grounding_sha256"]) == 64 + + +def test_solutions_resume_only_answers_missing_examples(tmp_path): + bundles = _bundles_file(tmp_path) + out = tmp_path / "resume-solutions" + first = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "no_grounding", + "--limit", + "1", + "--out", + str(out), + ) + assert first.returncode == 0, first.stderr + second = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "no_grounding", + "--limit", + "2", + "--resume", + "--out", + str(out), + ) + assert second.returncode == 0, second.stderr + records = [ + json.loads(line) + for line in (out / "solutions-no_grounding.jsonl").read_text().splitlines() + ] + assert len(records) == 2 + assert len({record["example_id"] for record in records}) == 2 + + +def test_solutions_resume_refuses_a_different_seed(tmp_path): + bundles = _bundles_file(tmp_path) + out = tmp_path / "incompatible-resume" + first = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "asdecided", + "--limit", + "1", + "--seed", + "0", + "--out", + str(out), + ) + assert first.returncode == 0, first.stderr + second = _run( + "run.py", + "solutions", + "--bundles", + bundles, + "--answering", + "offline-stub", + "--arms", + "asdecided", + "--limit", + "1", + "--seed", + "1", + "--resume", + "--out", + str(out), + ) + assert second.returncode == 2 + assert "cannot resume incompatible record" in second.stderr def test_solutions_refuse_real_backends_without_keys(tmp_path, monkeypatch): - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) # the subprocess inherits os.environ + monkeypatch.delenv( + "ANTHROPIC_API_KEY", raising=False + ) # the subprocess inherits os.environ completed = _run( - "run.py", "solutions", "--bundles", _bundles_file(tmp_path), - "--answering", "claude", "--out", str(tmp_path / "sol"), + "run.py", + "solutions", + "--bundles", + _bundles_file(tmp_path), + "--answering", + "claude", + "--out", + str(tmp_path / "sol"), ) assert completed.returncode != 0 assert "ANTHROPIC_API_KEY" in (completed.stderr + completed.stdout) @@ -194,17 +397,35 @@ def test_solutions_refuse_real_backends_without_keys(tmp_path, monkeypatch): def _score_records(tmp_path) -> list[dict]: records = tmp_path / "resolution_records.jsonl" completed = _run( - "run.py", "score", "--arm", "rac", "--eval-results", str(EVAL_RAC), - "--out", str(records), "--answering-model", "claude-opus-4-8", - "--upstream-harness", "test-commit", + "run.py", + "score", + "--arm", + "asdecided", + "--eval-results", + str(EVAL_ASDECIDED), + "--out", + str(records), + "--answering-model", + "claude-opus-4-8", + "--upstream-harness", + "test-commit", ) assert completed.returncode == 0, completed.stderr completed = _run( - "run.py", "score", "--arm", "no_grounding", "--eval-results", str(EVAL_NONE), - "--out", str(records), "--append", + "run.py", + "score", + "--arm", + "no_grounding", + "--eval-results", + str(EVAL_NONE), + "--out", + str(records), + "--append", ) assert completed.returncode == 0, completed.stderr - return [json.loads(line) for line in records.read_text(encoding="utf-8").splitlines()] + return [ + json.loads(line) for line in records.read_text(encoding="utf-8").splitlines() + ] def test_score_emits_schema_valid_paired_records(tmp_path): @@ -227,9 +448,39 @@ def test_stats_reproduces_the_hand_computed_mcnemar(tmp_path): completed = _run("run.py", "stats", "--records", str(records)) assert completed.returncode == 0, completed.stderr stats = json.loads(completed.stdout) - pair = stats["pairs"]["rac_vs_no_grounding"] - # fixture design: rac passes 3/3, no_grounding 1/3 -> a=1, b=2, c=0, d=0 + pair = stats["pairs"]["asdecided_vs_no_grounding"] + # fixture design: asdecided passes 3/3, no_grounding 1/3 -> a=1, b=2, c=0, d=0 assert pair["table"] == [1, 2, 0, 0] # exact two-sided binomial at min(2,0)=0 of 2 discordant: 2 * (1/4) = 0.5 assert pair["mcnemar"]["p_value"] == 0.5 assert pair["odds_ratio"]["degenerate"] is True + + +def test_stats_refuses_incomplete_required_arms(tmp_path): + records = tmp_path / "resolution_records.jsonl" + _score_records(tmp_path) + asdecided_ids = [ + json.loads(line)["example_id"] + for line in records.read_text(encoding="utf-8").splitlines() + if json.loads(line)["arm"] == "asdecided" + ] + removed_id = asdecided_ids[-1] + kept = [ + line + for line in records.read_text(encoding="utf-8").splitlines() + if not ( + json.loads(line)["arm"] == "asdecided" + and json.loads(line)["example_id"] == removed_id + ) + ] + records.write_text("\n".join(kept) + "\n", encoding="utf-8") + completed = _run( + "run.py", + "stats", + "--records", + str(records), + "--require-arms", + "no_grounding,asdecided", + ) + assert completed.returncode == 2 + assert "incomplete paired records" in completed.stderr