From d47ccc9920bc8a273dd183ae5cff036b7698caca Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Mon, 27 Jul 2026 08:10:16 +0100 Subject: [PATCH] feat(decisiongrounding): cut AsDecided arms over to Rust Core --- .github/workflows/autonomousqa.yml | 2 +- .github/workflows/decisiongrounding-ci.yml | 19 +++-- decisiongrounding/README.md | 13 ++-- .../docs/AGENTIC_BENCHMARK_RUN_HANDOFF.md | 2 +- decisiongrounding/providers/__init__.py | 4 +- decisiongrounding/providers/rac.py | 74 +++++++++--------- decisiongrounding/providers/rac_snippets.py | 7 +- decisiongrounding/pyproject.toml | 5 +- decisiongrounding/runner/cli.py | 18 ++--- decisiongrounding/runner/dashboard.py | 2 +- decisiongrounding/scoring/crossover.py | 16 ++-- decisiongrounding/scripts/report.py | 4 +- decisiongrounding/tests/test_hardening.py | 6 +- decisiongrounding/tests/test_parity_arms.py | 6 +- decisiongrounding/tests/test_rac_version.py | 77 +++++++++++-------- decisiongrounding/tests/test_real_backends.py | 4 +- .../tests/test_real_scenarios.py | 14 ++-- 17 files changed, 153 insertions(+), 120 deletions(-) diff --git a/.github/workflows/autonomousqa.yml b/.github/workflows/autonomousqa.yml index c8c13a6..d52ee29 100644 --- a/.github/workflows/autonomousqa.yml +++ b/.github/workflows/autonomousqa.yml @@ -96,4 +96,4 @@ jobs: pip install -e . bash ../.github/scripts/install-asdecided.sh 0.23.1 - name: Smoke — drive, compile, fidelity-gate, meter, re-score - run: RAC_BIN=decided python -m runner.cli smoke + run: DECIDED_BIN=decided python -m runner.cli smoke diff --git a/.github/workflows/decisiongrounding-ci.yml b/.github/workflows/decisiongrounding-ci.yml index 5bdc771..3acb1f2 100644 --- a/.github/workflows/decisiongrounding-ci.yml +++ b/.github/workflows/decisiongrounding-ci.yml @@ -3,7 +3,7 @@ name: decisiongrounding CI # The decisiongrounding benchmark arrived from its standalone repository with # its workflow at decisiongrounding/.github/workflows/ci.yml — a path GitHub # does not read in a subdirectory. This root workflow re-homes those same jobs -# (test matrix, offline demo smoke, rac corpus gate) unchanged in substance, +# (test matrix, offline demo smoke, live AsDecided arm smoke, corpus gate) # scoped to the subdirectory. on: @@ -48,10 +48,10 @@ jobs: - name: Smoke the offline demo run: python -m runner.cli demo - corpus: - # Dogfood RAC: enforce the benchmark's own knowledge corpus the same way - # the benchmark studies — validation, relationships, and review must pass. - name: rac corpus gate + asdecided-core: + # Exercise the actual Rust system-under-test boundary and enforce the + # benchmark's own knowledge corpus. + name: AsDecided Core integration runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -65,5 +65,14 @@ jobs: - name: Install AsDecided run: bash ../.github/scripts/install-asdecided.sh 0.23.1 + - name: Install benchmark + run: pip install -e ".[dev,schema]" + + - name: Smoke the AsDecided whole-artifact arm + run: python -m runner.cli run --arm rac --scenarios scenarios --out /tmp/dg-core-smoke + + - name: Smoke the AsDecided snippet arm + run: python -m runner.cli run --arm rac_snippets --scenarios scenarios --out /tmp/dg-core-snippets-smoke + - name: Gate the RAC corpus (validate + relationships + review) run: decided gate rac diff --git a/decisiongrounding/README.md b/decisiongrounding/README.md index 8689c4d..9fca0b3 100644 --- a/decisiongrounding/README.md +++ b/decisiongrounding/README.md @@ -136,7 +136,8 @@ pip install -e ".[real,schema,chart]" export ANTHROPIC_API_KEY=... # pinned answering model: claude-opus-4-8 export VOYAGE_API_KEY=... # real embeddings for naive_rag -# rac arm additionally needs the `rac` CLI on PATH (or set RAC_BIN) +# stable rac arm additionally needs AsDecided Core's `decided` CLI on PATH +# (or set DECIDED_BIN) python -m runner.cli compare \ --arms context_dump,naive_rag,rac \ --answering claude \ @@ -154,7 +155,7 @@ so a run says exactly what produced it. For repeat or large runs, `python -m runner.cli batch …` runs the same comparison through the **Message Batches API at ~50% of standard token price**. -It assembles every arm's grounding locally (rac CLI, embeddings) up front, then +It assembles every arm's grounding locally (AsDecided Core, embeddings) up front, then submits all answering calls as one batch and polls to completion (asynchronous — usually under an hour). The trade vs `compare` is the live, abortable per-cell feedback; for the first exploratory run prefer `compare`, for bulk runs prefer @@ -162,7 +163,7 @@ feedback; for the first exploratory run prefer `compare`, for bulk runs prefer A real run is expensive, so the runner protects your spend two ways. It **preflights** the configuration before doing any work — a missing -`ANTHROPIC_API_KEY`, `VOYAGE_API_KEY`, backend package, or `rac` CLI fails fast +`ANTHROPIC_API_KEY`, `VOYAGE_API_KEY`, backend package, or Core CLI fails fast with an actionable message instead of part-way through a paid sweep. And it **streams every completed run** to a durable `results/run--