From 1b0c807640b653a2f2caf22bebd359527f6d2d8a Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Wed, 7 Oct 2026 15:58:56 +0200 Subject: [PATCH 1/8] Fix Luna LM Checklist request formatting Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evaluator/scores.py | 36 +++++++++++++++++++++ tests/test_bceval_scores.py | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/test_bceval_scores.py diff --git a/evaluator/scores.py b/evaluator/scores.py index 569b54800..7c3096fc2 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -1,7 +1,43 @@ from __future__ import annotations +import importlib from typing import Any +_LUNA_JUDGE_MODEL = "gpt-56-reasoning-nano-luna" + + +def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + { + **message, + "content": [{"type": "output_text", "text": content}], + } + if message.get("role") == "assistant" and isinstance((content := message.get("content")), str) + else message + for message in messages + ] + + +class LmChecklist: + def __new__(cls) -> object: + # bc-eval loads a custom class with this name before its built-in scorer. + module = importlib.import_module("bc_eval.scorers.autoeval.lmchecklist") + scorer = module.LmChecklist() + request_args = scorer._request_args + + def responses_compatible_request_args(output: object, expected: object = None, **kwargs: object) -> dict[str, Any]: + request = request_args(output, expected, **kwargs) + model = str(request.get("model", "")) + # Autoevals routes GPT-5 models through Responses, where assistant content is output. + if model.startswith("gpt-5"): + request["messages"] = _responses_compatible_messages(request["messages"]) + if model == _LUNA_JUDGE_MODEL: + request.pop("temperature", None) + return request + + scorer._request_args = responses_compatible_request_args + return scorer + class ResolutionRate: def __call__(self, *, metadata: dict[str, Any], **kwargs: object) -> bool: diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py new file mode 100644 index 000000000..dadb89d53 --- /dev/null +++ b/tests/test_bceval_scores.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import sys +import types +from typing import cast + +import pytest + +from evaluator import scores + + +@pytest.mark.parametrize( + ("model", "expected_content", "expects_temperature"), + [ + ( + "gpt-56-reasoning-nano-luna", + [{"type": "output_text", "text": "answer"}], + False, + ), + ( + "gpt-5-mini", + [{"type": "output_text", "text": "answer"}], + True, + ), + ("gpt-41-2025-04-14", "answer", True), + ], +) +def test_lm_checklist_adapts_responses_payload_and_luna_parameters( + monkeypatch, + model, + expected_content, + expects_temperature, +): + class _BuiltInLmChecklist: + def _request_args(self, output, expected, **kwargs): + return { + "model": model, + "temperature": 0, + "messages": [ + {"role": "system", "content": "instructions"}, + {"role": "user", "content": "question"}, + {"role": "assistant", "content": output}, + ], + } + + module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") + module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, module.__name__, module) + + scorer = cast(_BuiltInLmChecklist, scores.LmChecklist()) + request = scorer._request_args("answer", {"assertions": []}) + + assert isinstance(scorer, _BuiltInLmChecklist) + assert request["messages"] == [ + {"role": "system", "content": "instructions"}, + {"role": "user", "content": "question"}, + { + "role": "assistant", + "content": expected_content, + }, + ] + assert ("temperature" in request) is expects_temperature From f2d39a0e4261bc114ae8fe5476f7ade367d5113a Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Wed, 7 Oct 2026 09:37:55 +0200 Subject: [PATCH 2/8] Fix skipped BCal aggregation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bcal-evaluation.yml | 54 +++++++++++++++-- .github/workflows/get-entries.yml | 6 +- .github/workflows/summarize-results.yml | 63 +++++++++++++++---- src/bcbench/commands/dataset.py | 9 ++- src/bcbench/commands/result.py | 76 +++++++++++++++++++++-- src/bcbench/results/__init__.py | 5 +- src/bcbench/results/bceval_export.py | 3 + src/bcbench/results/completeness.py | 59 ++++++++++++++++++ src/bcbench/results/display.py | 13 +++- src/bcbench/results/summary.py | 4 ++ tests/test_cli_commands.py | 81 +++++++++++++++++++++++++ tests/test_completeness.py | 24 ++++++++ tests/test_review_workflows.py | 37 +++++++++++ 13 files changed, 407 insertions(+), 27 deletions(-) create mode 100644 src/bcbench/results/completeness.py create mode 100644 tests/test_completeness.py diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml index 2b29d6b53..486e27c91 100644 --- a/.github/workflows/bcal-evaluation.yml +++ b/.github/workflows/bcal-evaluation.yml @@ -31,6 +31,7 @@ concurrency: env: EVALUATION_RESULTS_DIR: evaluation_results + BCCONTAINERHELPER_VERSION: 6.1.18 jobs: get-entries: @@ -39,9 +40,44 @@ jobs: test-run: ${{ inputs.test-run || false }} category: nl2al - evaluate-with-bcal: + prepare-bccontainerhelper: runs-on: windows-latest needs: get-entries + if: needs.get-entries.outputs.entries != '[]' + steps: + - name: Cache pinned BcContainerHelper + id: cache-bccontainerhelper + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~\Documents\PowerShell\Modules\BcContainerHelper + key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }} + + - name: Download pinned BcContainerHelper + if: steps.cache-bccontainerhelper.outputs.cache-hit != 'true' + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $moduleRoot = Join-Path $HOME 'Documents\PowerShell\Modules' + New-Item -ItemType Directory -Path $moduleRoot -Force | Out-Null + Save-Module ` + -Name BcContainerHelper ` + -RequiredVersion $env:BCCONTAINERHELPER_VERSION ` + -Repository PSGallery ` + -Path $moduleRoot ` + -Force ` + -ErrorAction Stop + + - name: Verify pinned BcContainerHelper + shell: pwsh + run: | + Import-Module BcContainerHelper -RequiredVersion $env:BCCONTAINERHELPER_VERSION -Force -DisableNameChecking + if ((Get-Module BcContainerHelper).Version -ne [Version]$env:BCCONTAINERHELPER_VERSION) { + throw "Expected BcContainerHelper $env:BCCONTAINERHELPER_VERSION." + } + + evaluate-with-bcal: + runs-on: windows-latest + needs: [get-entries, prepare-bccontainerhelper] outputs: results-dir: ${{ env.EVALUATION_RESULTS_DIR }} if: needs.get-entries.outputs.entries != '[]' @@ -136,6 +172,13 @@ jobs: id: bcversion run: uv run bcbench dataset version "${{ matrix.entry }}" --category nl2al --github-output version + - name: Restore pinned BcContainerHelper + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~\Documents\PowerShell\Modules\BcContainerHelper + key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }} + fail-on-cache-miss: true + # Cache BcContainerHelper's artifact folder (its hardcoded default path) keyed by this entry's BC version - name: Cache BC sandbox artifacts id: bccache @@ -148,7 +191,6 @@ jobs: if: steps.bccache.outputs.cache-hit != 'true' shell: pwsh run: | - Install-Module -Name BcContainerHelper -Force -AllowClobber -AllowPrerelease .\scripts\Download-BCSymbols.ps1 -Category nl2al -InstanceId "${{ matrix.entry }}" - name: Azure Login (OIDC) for CAPI Key Vault @@ -186,6 +228,7 @@ jobs: CAPI_CLIENT_ID: ${{ secrets.CAPI_CLIENT_ID }} CAPI_CERTIFICATE_KEY_VAULT_NAME: ${{ secrets.CAPI_CERTIFICATE_KEY_VAULT_NAME }} CAPI_CERTIFICATE_NAME: ${{ secrets.CAPI_CERTIFICATE_NAME }} + PYTHONPATH: ${{ github.workspace }}\src shell: pwsh run: | $env:CAPI_CERT_FILE = "$env:RUNNER_TEMP\capi-cert.pfx" @@ -200,12 +243,13 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: - name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }} + name: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.entry }} path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: - needs: evaluate-with-bcal + needs: [get-entries, evaluate-with-bcal] + if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }} uses: $/.github/workflows/summarize-results.yml permissions: contents: write @@ -217,4 +261,6 @@ jobs: mock: ${{ inputs.test-run || false }} category: nl2al skip-leaderboard: true + expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }} + artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-* secrets: inherit diff --git a/.github/workflows/get-entries.yml b/.github/workflows/get-entries.yml index b49e08d86..7f73d6907 100644 --- a/.github/workflows/get-entries.yml +++ b/.github/workflows/get-entries.yml @@ -24,6 +24,9 @@ on: entries: description: JSON array of dataset entries value: ${{ jobs.get-entries.outputs.entries }} + entry-count: + description: Number of dataset entries + value: ${{ jobs.get-entries.outputs.entry-count }} runner: description: GitHub Actions runner label to evaluate this category on value: ${{ jobs.get-entries.outputs.runner }} @@ -39,6 +42,7 @@ jobs: runs-on: ubuntu-latest outputs: entries: ${{ steps.get-entries.outputs.entries }} + entry-count: ${{ steps.get-entries.outputs.entry-count }} runner: ${{ steps.runtime-config.outputs.runner }} requires-container: ${{ steps.runtime-config.outputs.requires-container }} requires-repo: ${{ steps.runtime-config.outputs.requires-repo }} @@ -58,7 +62,7 @@ jobs: - name: Get entries for matrix id: get-entries run: | - cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries" + cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries --count-output entry-count" if [[ "${{ inputs.modified-only }}" == "true" ]]; then cmd="$cmd --modified-only" diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index d77dd3861..fa0eea50b 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -34,10 +34,21 @@ on: required: false type: string default: "" + expected-total: + description: "Expected unique matrix entry count; negative disables completeness enforcement" + required: false + type: number + default: -1 + artifact-pattern: + description: "Artifact glob to download" + required: false + type: string + default: "*" env: BCEVAL_RESULT_FILE: bceval_results.jsonl SUMMARY_OUTPUT_FILE: evaluation_summary.json + COMPLETENESS_OUTPUT_FILE: evaluation_completeness.json jobs: summarize-results: @@ -58,22 +69,27 @@ jobs: - name: Download all evaluation results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: ${{ inputs.expected-total >= 0 }} with: path: ${{ inputs.results-dir }}/ + pattern: ${{ inputs.artifact-pattern }} merge-multiple: true - name: Summarize evaluation results - run: uv run bcbench result summarize --category "${{ inputs.category }}" --result-dir "${{ inputs.results-dir }}" --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" --git-ref "${{ inputs.git-ref || github.ref_name }}" - - - name: Upload evaluation summary to artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: evaluation-summary - path: ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }} - if-no-files-found: error - retention-days: ${{ inputs.mock && 1 || 30 }} + id: summarize + run: >- + uv run bcbench result summarize + --category "${{ inputs.category }}" + --result-dir "${{ inputs.results-dir }}" + --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" + --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" + --completeness-output "${{ env.COMPLETENESS_OUTPUT_FILE }}" + --expected-total "${{ inputs.expected-total }}" + --git-ref "${{ inputs.git-ref || github.ref_name }}" + --github-output - name: Azure Login with OIDC for bceval package + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} @@ -81,10 +97,12 @@ jobs: allow-no-subscriptions: true - name: Resolve bceval evaluator config for category + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' id: bceval run: uv run bcbench category bceval-config --category "${{ inputs.category }}" - - name: Upload result using bceval + - name: Score and upload complete result using bceval + if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true' env: BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} @@ -96,7 +114,6 @@ jobs: uv tool install bc-eval[capi]==0.3.14 --python 3.12 --index "https://anything:${ADO_TOKEN}@dynamicssmb2.pkgs.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_packaging/BC_PythonFeed/pypi/simple/" - # Upload summary using bc-eval MODEL_TAG="${{ inputs.model }}" MODEL_TAG="${MODEL_TAG//./-}" BRANCH_TAG="${{ inputs.git-ref || github.ref_name }}" @@ -115,8 +132,20 @@ jobs: --metrics "bc_bench_metrics" \ --use-capi ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }} + - name: Upload evaluation summary to artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evaluation-summary-${{ github.run_attempt }} + path: | + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }} + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }} + ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }} + if-no-files-found: error + retention-days: ${{ inputs.mock && 1 || 30 }} + - name: Update leaderboard in a new branch - if: ${{ !inputs.mock && !inputs.skip-leaderboard }} + if: ${{ !inputs.mock && !inputs.skip-leaderboard && steps.summarize.outputs.complete == 'true' }} run: | git fetch origin main @@ -150,3 +179,13 @@ jobs: git commit -m "Update leaderboard: ${{ inputs.agent }} (${{ inputs.model }}) - Run ${{ github.run_id }}" git push --set-upstream origin $branchName + + - name: Require complete matrix + if: ${{ always() && inputs.expected-total >= 0 }} + run: | + completeness_file="${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}" + if [[ ! -f "$completeness_file" ]]; then + echo "::error::Completeness metadata was not produced." + exit 1 + fi + uv run bcbench result require-complete "$completeness_file" diff --git a/src/bcbench/commands/dataset.py b/src/bcbench/commands/dataset.py index 3f3092749..a3e904d9f 100644 --- a/src/bcbench/commands/dataset.py +++ b/src/bcbench/commands/dataset.py @@ -21,6 +21,7 @@ def list_entries( category: EvaluationCategoryOption = EvaluationCategory.BUG_FIX, github_output: Annotated[str | None, typer.Option(help="Write JSON output to GITHUB_OUTPUT with this key name")] = None, + count_output: Annotated[str | None, typer.Option(help="Write the entry count to GITHUB_OUTPUT with this key name")] = None, modified_only: Annotated[bool, typer.Option(help="Only list entries that have been modified in git diff")] = False, test_run: Annotated[bool, typer.Option(help="Indicate this is a test run (with 2 entries)")] = False, ) -> None: @@ -58,8 +59,12 @@ def list_entries( for entry_id in entry_ids: print(f" - {entry_id}") - if github_output: - write_step_outputs({github_output: json.dumps(entry_ids)}) + step_outputs = { + **({github_output: json.dumps(entry_ids)} if github_output else {}), + **({count_output: str(len(entry_ids))} if count_output else {}), + } + if step_outputs: + write_step_outputs(step_outputs) @dataset_app.command("view") diff --git a/src/bcbench/commands/result.py b/src/bcbench/commands/result.py index 71e646c7a..bb374ab1b 100644 --- a/src/bcbench/commands/result.py +++ b/src/bcbench/commands/result.py @@ -9,12 +9,15 @@ from bcbench.cli_options import EvaluationCategoryOption, OutputDir, RunId from bcbench.config import get_config +from bcbench.github_actions import write_step_outputs from bcbench.results import ( BaseEvaluationResult, + EvaluationCompleteness, EvaluationResultSummary, Leaderboard, LeaderboardAggregate, create_console_summary, + create_github_completeness_summary, create_github_job_summary, write_bceval_results, ) @@ -36,6 +39,9 @@ def result_summarize( summary_output: Annotated[str, typer.Option(help="Output filename for summary JSON")] = "evaluation_summary.json", bceval_output: Annotated[str, typer.Option(help="Output filename for bceval results")] = "bceval_results.jsonl", git_ref: Annotated[str | None, typer.Option("--git-ref", help="Git ref (branch/tag) the run was dispatched from; recorded in bceval metadata as git_branch")] = None, + expected_total: Annotated[int, typer.Option(help="Expected number of unique matrix entries; negative disables completeness tracking")] = -1, + completeness_output: Annotated[str, typer.Option(help="Output filename for matrix completeness metadata")] = "evaluation_completeness.json", + github_output: Annotated[bool, typer.Option(help="Write produced/expected/complete step outputs to GITHUB_OUTPUT")] = False, ) -> None: """ Summarize evaluation results from a completed run. @@ -43,13 +49,16 @@ def result_summarize( Aggregates individual instance results, displays job summaries and generates bceval output format. """ run_dir: Path = result_dir / run_id + track_completeness = expected_total >= 0 if not run_dir.exists(): - logger.error(f"Results directory not found: {run_dir}") - raise typer.Exit(code=1) + if not track_completeness: + logger.error(f"Results directory not found: {run_dir}") + raise typer.Exit(code=1) + run_dir.mkdir(parents=True) result_files = list(run_dir.rglob(result_pattern)) - if not result_files: + if not result_files and not track_completeness: logger.error(f"No result files matching '{result_pattern}' found in {run_dir}") raise typer.Exit(code=1) @@ -57,7 +66,7 @@ def result_summarize( instance_pattern_regex = re.compile(_config.file_patterns.instance_pattern) result_files = [f for f in result_files if instance_pattern_regex.match(f.stem)] - if not result_files: + if not result_files and not track_completeness: logger.error(f"No instance-specific result files found in {run_dir}") raise typer.Exit(code=1) @@ -67,13 +76,44 @@ def result_summarize( with results_path.open() as f: results.extend(BaseEvaluationResult.from_json(json.loads(line)) for line in f if line.strip()) - if not results: + if not results and not track_completeness: logger.error("No results found in the result files") raise typer.Exit(code=1) - write_bceval_results(results, run_dir, run_id, bceval_output, category, git_ref=git_ref) + completeness = EvaluationCompleteness.from_instance_ids(expected_total, (result.instance_id for result in results)) if track_completeness else None + if completeness: + completeness.save(run_dir / completeness_output) + logger.info( + "Evaluation completeness: expected=%d produced=%d missing=%d duplicates=%d", + completeness.expected_entry_count, + completeness.produced_entry_count, + completeness.missing_entry_count, + completeness.duplicate_result_count, + ) + if _config.env.github_actions and not results: + create_github_completeness_summary(completeness) + + if github_output: + output_completeness = completeness or EvaluationCompleteness.from_instance_ids(len({result.instance_id for result in results}), (result.instance_id for result in results)) + write_step_outputs( + { + "expected": str(output_completeness.expected_entry_count), + "produced": str(output_completeness.produced_entry_count), + "missing": str(output_completeness.missing_entry_count), + "complete": str(output_completeness.complete).lower(), + "has-results": str(bool(results)).lower(), + } + ) + + if not results: + logger.error("No matrix shard produced an evaluation result.") + return + + write_bceval_results(results, run_dir, run_id, bceval_output, category, git_ref=git_ref, completeness=completeness) summary = EvaluationResultSummary.from_results(results, run_id=run_id) + if completeness: + summary = summary.model_copy(update={"completeness": completeness}) if _config.env.github_actions: create_github_job_summary(results, summary) @@ -83,6 +123,30 @@ def result_summarize( summary.save(run_dir, summary_output) +@result_app.command("require-complete") +def result_require_complete( + completeness_file: Annotated[Path, typer.Argument(exists=True, file_okay=True, dir_okay=False)], +) -> None: + completeness = EvaluationCompleteness.load(completeness_file) + if completeness.complete: + logger.info( + "Evaluation is complete: %d/%d entries produced.", + completeness.produced_entry_count, + completeness.expected_entry_count, + ) + return + + logger.error( + "Evaluation is incomplete: expected=%d produced=%d missing=%d unexpected=%d duplicates=%d", + completeness.expected_entry_count, + completeness.produced_entry_count, + completeness.missing_entry_count, + completeness.unexpected_entry_count, + completeness.duplicate_result_count, + ) + raise typer.Exit(code=1) + + def _rebuild_aggregates(runs: list[EvaluationResultSummary]) -> list[LeaderboardAggregate]: grouped: defaultdict[tuple[str | None, ...], list[EvaluationResultSummary]] = defaultdict(list) for run in runs: diff --git a/src/bcbench/results/__init__.py b/src/bcbench/results/__init__.py index 1d8112d0d..6971ae82a 100644 --- a/src/bcbench/results/__init__.py +++ b/src/bcbench/results/__init__.py @@ -1,7 +1,8 @@ from bcbench.results.base import ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.results.bceval_export import write_bceval_results from bcbench.results.codereview import CodeReviewResult, CodeReviewResultSummary -from bcbench.results.display import create_console_summary, create_github_job_summary +from bcbench.results.completeness import EvaluationCompleteness +from bcbench.results.display import create_console_summary, create_github_completeness_summary, create_github_job_summary from bcbench.results.leaderboard import ( CodeReviewLeaderboardAggregate, ExecutionBasedLeaderboardAggregate, @@ -20,6 +21,7 @@ "CodeReviewLeaderboardAggregate", "CodeReviewResult", "CodeReviewResultSummary", + "EvaluationCompleteness", "EvaluationResultSummary", "ExecutionBasedEvaluationResult", "ExecutionBasedEvaluationResultSummary", @@ -29,6 +31,7 @@ "Leaderboard", "LeaderboardAggregate", "create_console_summary", + "create_github_completeness_summary", "create_github_job_summary", "write_bceval_results", ] diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index a5e76f820..12a93d9d2 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -9,6 +9,7 @@ from bcbench.dataset import BaseDatasetEntry from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.results.summary import get_benchmark_version from bcbench.types import EvaluationCategory, ExpectedOutput, ExperimentConfiguration @@ -37,6 +38,7 @@ def write_bceval_results( output_filename: str, category: EvaluationCategory, git_ref: str | None = None, + completeness: EvaluationCompleteness | None = None, ) -> None: """Write results into a JSONL file for bceval consumption.""" entry_cls = category.entry_class @@ -72,6 +74,7 @@ def write_bceval_results( "error_message": result.error_message, "tool_usage": (result.metrics.tool_usage if result.metrics and result.metrics.tool_usage else None) or 0, **_experiment_metadata(result.experiment, git_ref, benchmark_version), + **(completeness.to_metadata() if completeness else {}), } bceval_result = { diff --git a/src/bcbench/results/completeness.py b/src/bcbench/results/completeness.py new file mode 100644 index 000000000..d23563814 --- /dev/null +++ b/src/bcbench/results/completeness.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import json +from collections.abc import Iterable +from pathlib import Path + +from pydantic import BaseModel, Field + + +class EvaluationCompleteness(BaseModel): + expected_entry_count: int = Field(ge=0) + produced_entry_count: int = Field(ge=0) + missing_entry_count: int = Field(ge=0) + unexpected_entry_count: int = Field(ge=0) + duplicate_result_count: int = Field(ge=0) + complete: bool + + @classmethod + def from_instance_ids(cls, expected_entry_count: int, instance_ids: Iterable[str]) -> EvaluationCompleteness: + ids = list(instance_ids) + produced_entry_count = len(set(ids)) + return cls( + expected_entry_count=expected_entry_count, + produced_entry_count=produced_entry_count, + missing_entry_count=max(expected_entry_count - produced_entry_count, 0), + unexpected_entry_count=max(produced_entry_count - expected_entry_count, 0), + duplicate_result_count=len(ids) - produced_entry_count, + complete=produced_entry_count == expected_entry_count and len(ids) == produced_entry_count, + ) + + def to_metadata(self) -> dict[str, int | bool]: + return { + "expected_entry_count": self.expected_entry_count, + "produced_entry_count": self.produced_entry_count, + "missing_entry_count": self.missing_entry_count, + "unexpected_entry_count": self.unexpected_entry_count, + "duplicate_result_count": self.duplicate_result_count, + "evaluation_complete": self.complete, + } + + def save(self, output_file: Path) -> None: + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(self.model_dump_json(indent=2), encoding="utf-8") + + @classmethod + def load(cls, input_file: Path) -> EvaluationCompleteness: + return cls.model_validate(json.loads(input_file.read_text(encoding="utf-8"))) + + def render_markdown(self) -> str: + status = ":white_check_mark: Complete" if self.complete else ":x: Incomplete" + return "\n".join( + [ + "## Evaluation completeness", + "", + "| Expected entries | Produced entries | Missing/failed entries | Unexpected entries | Duplicate results | Status |", + "|---:|---:|---:|---:|---:|:---|", + (f"| {self.expected_entry_count} | {self.produced_entry_count} | {self.missing_entry_count} | {self.unexpected_entry_count} | {self.duplicate_result_count} | {status} |"), + ] + ) diff --git a/src/bcbench/results/display.py b/src/bcbench/results/display.py index f7cf3b3c6..a9d2e9cde 100644 --- a/src/bcbench/results/display.py +++ b/src/bcbench/results/display.py @@ -7,6 +7,7 @@ from bcbench.config import get_config from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.results.summary import EvaluationResultSummary, calculate_average_tool_usage logger = logging.getLogger(__name__) @@ -24,6 +25,8 @@ def _status_style(status_label: str) -> tuple[str, str]: def create_console_summary(results: Sequence[BaseEvaluationResult], summary: EvaluationResultSummary) -> None: console.print("\n[bold cyan]Evaluation Results Summary[/bold cyan]") + if summary.completeness: + console.print(f"Completeness: [bold]{summary.completeness.produced_entry_count}/{summary.completeness.expected_entry_count}[/bold] ({summary.completeness.missing_entry_count} missing/failed)") console.print(f"Total Processed: [bold]{len(results)}[/bold], using [bold]{results[0].agent_name}({results[0].model})[/bold]") console.print(f"Agent Version: [bold]{results[0].agent_version or 'Unrecorded'}[/bold]") console.print(f"Category: [bold]{results[0].category.value}[/bold]") @@ -80,6 +83,7 @@ def _get_short_error_message(error_message: str | None) -> str: def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: EvaluationResultSummary) -> None: metrics_section: str = summary.render_github_metrics_markdown().strip() + completeness_section = summary.completeness.render_markdown() if summary.completeness else "" # Calculate average tool usage tool_usage_section: str = "" @@ -104,7 +108,10 @@ def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: f"- Plugins: {', '.join(results[0].experiment.plugins) if results[0].experiment and results[0].experiment.plugins else 'None'}", ] ) - sections: list[str] = [header_section, *(section for section in [metrics_section, tool_usage_section] if section), "## Detailed Results\n\n"] + sections: list[str] = [ + *(section for section in [completeness_section, header_section, metrics_section, tool_usage_section] if section), + "## Detailed Results\n\n", + ] markdown_summary: str = "\n\n".join(sections) # Dynamic columns from display_row() @@ -132,6 +139,10 @@ def create_github_job_summary(results: Sequence[BaseEvaluationResult], summary: _write_github_step_summary(markdown_summary) +def create_github_completeness_summary(completeness: EvaluationCompleteness) -> None: + _write_github_step_summary(completeness.render_markdown()) + + def _write_github_step_summary(content: str) -> None: config = get_config() if config.env.github_step_summary: diff --git a/src/bcbench/results/summary.py b/src/bcbench/results/summary.py index 0aa202779..2c0b3e2a6 100644 --- a/src/bcbench/results/summary.py +++ b/src/bcbench/results/summary.py @@ -12,6 +12,7 @@ from pydantic import BaseModel, Field from bcbench.results.base import BaseEvaluationResult +from bcbench.results.completeness import EvaluationCompleteness from bcbench.types import EvaluationCategory, ExperimentConfiguration if TYPE_CHECKING: @@ -58,6 +59,7 @@ class EvaluationResultSummary(BaseModel, ABC): github_run_id: str | None = None experiment: ExperimentConfiguration | None = None + completeness: EvaluationCompleteness | None = None benchmark_version: str @@ -131,6 +133,8 @@ def to_dict(self) -> dict[str, Any]: data["average_llm_duration"] = round(data["average_llm_duration"], 1) if data["average_llm_duration"] is not None else None if data["average_tool_usage"] is None: del data["average_tool_usage"] + if data["completeness"] is None: + del data["completeness"] return data def save(self, output_dir: Path, summary_file: str) -> None: diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index b97d2c856..34942273c 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -432,6 +432,75 @@ def test_result_summarize_with_custom_pattern(sample_results_directory, problem_ assert result.exit_code == 0 +@pytest.mark.integration +def test_result_summarize_records_partial_matrix_completeness(sample_results_directory, problem_statement_dir, tmp_path, monkeypatch): + base_path, run_id, dataset_path = sample_results_directory + github_output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + + with ( + patch.object(_BugFixTestGenBase, "problem_statement_dir", property(lambda self: problem_statement_dir)), + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=dataset_path), + ): + result = runner.invoke( + app, + [ + "result", + "summarize", + "--category", + "bug-fix", + "--run-id", + run_id, + "--result-dir", + str(base_path), + "--expected-total", + "4", + "--github-output", + ], + ) + + assert result.exit_code == 0, result.exception + run_dir = base_path / run_id + completeness = json.loads((run_dir / "evaluation_completeness.json").read_text()) + assert completeness["expected_entry_count"] == 4 + assert completeness["produced_entry_count"] == 3 + assert completeness["missing_entry_count"] == 1 + assert completeness["complete"] is False + + summary = json.loads((run_dir / "evaluation_summary.json").read_text()) + assert summary["completeness"] == completeness + exported = [json.loads(line) for line in (run_dir / "bceval_results.jsonl").read_text().splitlines()] + assert {row["metadata"]["evaluation_complete"] for row in exported} == {False} + assert "complete=false" in github_output.read_text() + + +@pytest.mark.integration +def test_result_summarize_zero_results_still_writes_completeness(tmp_path): + run_id = "empty-matrix" + result = runner.invoke( + app, + [ + "result", + "summarize", + "--category", + "nl2al", + "--run-id", + run_id, + "--result-dir", + str(tmp_path), + "--expected-total", + "110", + ], + ) + + assert result.exit_code == 0 + completeness = json.loads((tmp_path / run_id / "evaluation_completeness.json").read_text()) + assert completeness["produced_entry_count"] == 0 + assert completeness["missing_entry_count"] == 110 + assert completeness["complete"] is False + assert not (tmp_path / run_id / "evaluation_summary.json").exists() + + @pytest.mark.integration def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): @@ -450,6 +519,18 @@ def test_dataset_list_displays_all_entries(sample_dataset_file_for_cli): assert "Found 3 entry(ies)" in result.stdout +@pytest.mark.integration +def test_dataset_list_writes_entry_count_output(sample_dataset_file_for_cli, tmp_path, monkeypatch): + github_output = tmp_path / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + + with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): + result = runner.invoke(app, ["dataset", "list", "--count-output", "entry-count"]) + + assert result.exit_code == 0 + assert github_output.read_text() == "entry-count=3\n" + + @pytest.mark.integration def test_dataset_bc_version_returns_entry_version(sample_dataset_file_for_cli): with patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=sample_dataset_file_for_cli): diff --git a/tests/test_completeness.py b/tests/test_completeness.py new file mode 100644 index 000000000..58f3bf015 --- /dev/null +++ b/tests/test_completeness.py @@ -0,0 +1,24 @@ +from bcbench.results.completeness import EvaluationCompleteness + + +def test_completeness_counts_unique_missing_and_duplicate_results(): + completeness = EvaluationCompleteness.from_instance_ids(4, ["a", "b", "b"]) + + assert completeness.expected_entry_count == 4 + assert completeness.produced_entry_count == 2 + assert completeness.missing_entry_count == 2 + assert completeness.duplicate_result_count == 1 + assert completeness.complete is False + + +def test_completeness_metadata_is_explicit_for_external_storage(): + completeness = EvaluationCompleteness.from_instance_ids(2, ["a", "b"]) + + assert completeness.to_metadata() == { + "expected_entry_count": 2, + "produced_entry_count": 2, + "missing_entry_count": 0, + "unexpected_entry_count": 0, + "duplicate_result_count": 0, + "evaluation_complete": True, + } diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 3f58a14a0..ce1096b78 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -203,3 +203,40 @@ def test_agent_workflows_select_al_tool_dotnet_version_for_bc_version() -> None: for workflow_name in ("claude-evaluation.yml", "copilot-evaluation.yml"): workflow = _workflow(workflow_name) assert '--framework "net${{ steps.setup-env.outputs.al_tool_dotnet_version }}"' in workflow + + +def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> None: + workflow = yaml.safe_load(_workflow("bcal-evaluation.yml")) + summarize = workflow["jobs"]["summarize-results"] + + assert summarize["needs"] == ["get-entries", "evaluate-with-bcal"] + assert "always()" in summarize["if"] + assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" + assert summarize["with"]["results-dir"] == "evaluation_results" + assert summarize["with"]["artifact-pattern"] == "evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*" + + +def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: + workflow_text = _workflow("bcal-evaluation.yml") + workflow = yaml.safe_load(workflow_text) + prepare_steps = workflow["jobs"]["prepare-bccontainerhelper"]["steps"] + download_step = next(step for step in prepare_steps if step["name"] == "Download pinned BcContainerHelper") + download_script = download_step["run"] + + assert "BCCONTAINERHELPER_VERSION: 6.1.18" in workflow_text + assert download_script.count("Save-Module") == 1 + assert "RequiredVersion $env:BCCONTAINERHELPER_VERSION" in download_script + assert "fail-on-cache-miss: true" in workflow_text + assert "Install-Module -Name BcContainerHelper" not in workflow_text + + +def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> None: + workflow = _workflow("summarize-results.yml") + + assert "continue-on-error: ${{ inputs.expected-total >= 0 }}" in workflow + assert "evaluation_completeness.json" in workflow + assert "bceval_results.jsonl" in workflow + assert "steps.summarize.outputs.complete == 'true'" in workflow + assert "bcbench.bceval_runner" not in workflow + assert "if: always()" in workflow + assert "bcbench result require-complete" in workflow From a63b91dacf6be4ce73b1d04bfc591b7fa6b80912 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Wed, 7 Oct 2026 13:14:11 +0200 Subject: [PATCH 3/8] Fix Luna LM Checklist response parsing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evaluator/scores.py | 110 ++++++++++++++++ tests/test_bceval_scores.py | 249 +++++++++++++++++++++++++++++++++--- 2 files changed, 339 insertions(+), 20 deletions(-) diff --git a/evaluator/scores.py b/evaluator/scores.py index 7c3096fc2..f9ed87181 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -1,9 +1,12 @@ from __future__ import annotations import importlib +from collections.abc import Mapping +from copy import copy from typing import Any _LUNA_JUDGE_MODEL = "gpt-56-reasoning-nano-luna" +_LUNA_RESPONSE_FORMAT = {"type": "json_object"} def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -18,6 +21,112 @@ def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[ ] +def _as_mapping(value: object) -> Mapping[str, Any] | None: + if isinstance(value, Mapping): + return value + for method_name in ("model_dump", "dict"): + method = getattr(value, method_name, None) + if callable(method): + mapped = method() + if isinstance(mapped, Mapping): + return mapped + return None + + +def _luna_output_text(response: object) -> tuple[Mapping[str, Any], str]: + response_data = _as_mapping(response) + if response_data is None: + raise TypeError(f"Luna Responses API returned unsupported response type {type(response).__name__}") + + text_parts: list[str] = [] + output = response_data.get("output") + if isinstance(output, list): + for raw_item in output: + item = _as_mapping(raw_item) + if item is None: + continue + item_type = item.get("type") + if item_type == "message" and item.get("role") == "assistant": + content = item.get("content") + if not isinstance(content, list): + continue + for raw_part in content: + part = _as_mapping(raw_part) + if part is None or part.get("type") not in ("output_text", "text"): + continue + text = part.get("text") + if isinstance(text, str): + text_parts.append(text) + elif item_type in ("output_text", "text"): + text = item.get("text") or item.get("content") + if isinstance(text, str): + text_parts.append(text) + + if not text_parts: + raise ValueError("Luna Responses API response contained no assistant output_text content") + return response_data, "".join(text_parts) + + +def _luna_chat_completion(response: object) -> dict[str, Any]: + response_data, content = _luna_output_text(response) + return { + "id": response_data.get("id"), + "object": "chat.completion", + "created": response_data.get("created_at", response_data.get("created")), + "model": response_data.get("model"), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": response_data.get("stop_reason", "stop"), + } + ], + } + + +def _luna_responses_params(request: dict[str, object]) -> dict[str, object]: + supported = { + "max_tokens", + "messages", + "model", + "reasoning_effort", + "response_format", + "span_info", + } + unsupported = sorted(set(request) - supported) + if unsupported: + raise ValueError(f"Unsupported Luna LM Checklist request arguments: {', '.join(unsupported)}") + if request.get("model") != _LUNA_JUDGE_MODEL: + raise ValueError("Luna Responses adapter received a non-Luna model") + + params = { + "model": request["model"], + "input": request["messages"], + } + if (max_tokens := request.get("max_tokens")) is not None: + params["max_output_tokens"] = max_tokens + if (response_format := request.get("response_format")) is not None: + if response_format != _LUNA_RESPONSE_FORMAT: + raise ValueError(f"Unsupported Luna LM Checklist response format: {response_format!r}") + params["text"] = {"format": dict(_LUNA_RESPONSE_FORMAT)} + if (reasoning_effort := request.get("reasoning_effort")) is not None: + params["reasoning"] = {"effort": reasoning_effort} + return params + + +def _luna_responses_client(client: object) -> object: + oai = importlib.import_module("autoevals.oai") + resolved_client = oai.prepare_openai(client=client, is_async=False) + adapted_client = copy(resolved_client) + + def complete(**request: object) -> dict[str, Any]: + response = resolved_client.openai.responses.create(**_luna_responses_params(request)) + return _luna_chat_completion(response) + + adapted_client.complete = complete + return adapted_client + + class LmChecklist: def __new__(cls) -> object: # bc-eval loads a custom class with this name before its built-in scorer. @@ -33,6 +142,7 @@ def responses_compatible_request_args(output: object, expected: object = None, * request["messages"] = _responses_compatible_messages(request["messages"]) if model == _LUNA_JUDGE_MODEL: request.pop("temperature", None) + request["client"] = _luna_responses_client(request.get("client")) return request scorer._request_args = responses_compatible_request_args diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py index dadb89d53..3a376e948 100644 --- a/tests/test_bceval_scores.py +++ b/tests/test_bceval_scores.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import sys import types from typing import cast @@ -9,48 +10,120 @@ from evaluator import scores +class _RecordingClient: + def __init__(self, *, responses_result=None, chat_result=None): + self.responses_result = responses_result + self.chat_result = chat_result + self.responses_calls = [] + self.chat_calls = [] + self.openai = types.SimpleNamespace(responses=types.SimpleNamespace(create=self._create_response)) + + def _create_response(self, **kwargs): + self.responses_calls.append(kwargs) + return self.responses_result + + def complete(self, **kwargs): + self.chat_calls.append(kwargs) + if self.chat_result is None: + raise AssertionError("Chat completions should not be used") + return self.chat_result + + +class _ModelResponse: + def __init__(self, data): + self.output = data["output"] + self._data = data + + def model_dump(self): + return self._data + + +def _install_lm_checklist_dependencies(monkeypatch, *, model, client): + class _BuiltInLmChecklist: + def __init__(self): + self.client = client + self.extra_args = {} + + def _request_args(self, output, expected, **kwargs): + return { + "client": self.client, + "model": model, + "temperature": 0, + "max_tokens": 4096, + "response_format": {"type": "json_object"}, + "messages": [ + {"role": "system", "content": "instructions"}, + {"role": "user", "content": "question"}, + {"role": "assistant", "content": output}, + ], + } + + def _process_response(self, response, assertions): + evaluated = json.loads(response["choices"][0]["message"]["content"])["assertions"] + passed = all( + next( + (result["pass"] for result in evaluated if result["text"] == assertion["text"]), + False, + ) + for assertion in assertions + ) + return [ + types.SimpleNamespace(name="pass_rate", score=float(passed)), + types.SimpleNamespace(name="test_passed", score=float(passed)), + ] + + def _run_eval_sync(self, output, expected, **kwargs): + request = self._request_args(output, expected, **kwargs) + request_client = prepare_openai(client=request.pop("client")) + response = request_client.complete(**request) + return self._process_response(response, expected["assertions"]) + + def prepare_openai(*, client=None, is_async=False): + assert not is_async + return client or _RecordingClient() + + bc_eval_module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") + bc_eval_module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, bc_eval_module.__name__, bc_eval_module) + + autoevals_module = types.ModuleType("autoevals.oai") + autoevals_module.prepare_openai = prepare_openai # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, autoevals_module.__name__, autoevals_module) + return _BuiltInLmChecklist + + @pytest.mark.parametrize( - ("model", "expected_content", "expects_temperature"), + ("model", "expected_content", "expects_temperature", "adapts_client"), [ ( "gpt-56-reasoning-nano-luna", [{"type": "output_text", "text": "answer"}], False, + True, ), ( "gpt-5-mini", [{"type": "output_text", "text": "answer"}], True, + False, ), - ("gpt-41-2025-04-14", "answer", True), + ("gpt-41-2025-04-14", "answer", True, False), ], ) -def test_lm_checklist_adapts_responses_payload_and_luna_parameters( +def test_lm_checklist_adapts_only_luna_responses_payload( monkeypatch, model, expected_content, expects_temperature, + adapts_client, ): - class _BuiltInLmChecklist: - def _request_args(self, output, expected, **kwargs): - return { - "model": model, - "temperature": 0, - "messages": [ - {"role": "system", "content": "instructions"}, - {"role": "user", "content": "question"}, - {"role": "assistant", "content": output}, - ], - } + client = _RecordingClient() + built_in = _install_lm_checklist_dependencies(monkeypatch, model=model, client=client) - module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") - module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] - monkeypatch.setitem(sys.modules, module.__name__, module) - - scorer = cast(_BuiltInLmChecklist, scores.LmChecklist()) + scorer = cast(built_in, scores.LmChecklist()) request = scorer._request_args("answer", {"assertions": []}) - assert isinstance(scorer, _BuiltInLmChecklist) + assert isinstance(scorer, built_in) assert request["messages"] == [ {"role": "system", "content": "instructions"}, {"role": "user", "content": "question"}, @@ -60,3 +133,139 @@ def _request_args(self, output, expected, **kwargs): }, ] assert ("temperature" in request) is expects_temperature + assert (request["client"] is not client) is adapts_client + + +def test_lm_checklist_parses_nested_luna_response(monkeypatch): + assertion = {"text": "The answer is correct.", "level": "critical"} + judgment = { + "assertions": [ + { + "text": assertion["text"], + "reasoning": "The response matches.", + "pass": True, + } + ] + } + client = _RecordingClient( + responses_result=_ModelResponse( + { + "id": "resp-1", + "object": "response", + "created_at": 123, + "model": "gpt-56-reasoning-nano-luna", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": json.dumps(judgment), + } + ], + } + ], + } + ) + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync( + "answer", + {"assertions": [assertion]}, + input="question", + ) + + assert {score.name: score.score for score in result} == { + "pass_rate": 1.0, + "test_passed": 1.0, + } + assert len(client.responses_calls) == 1 + responses_request = client.responses_calls[0] + assert responses_request["model"] == "gpt-56-reasoning-nano-luna" + assert responses_request["max_output_tokens"] == 4096 + assert responses_request["text"] == {"format": {"type": "json_object"}} + assert responses_request["input"][-1] == { + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + } + assert "max_tokens" not in responses_request + assert "response_format" not in responses_request + assert "temperature" not in responses_request + assert client.chat_calls == [] + + +def test_lm_checklist_delegates_non_luna_requests(monkeypatch): + assertion = {"text": "The answer is correct."} + client = _RecordingClient( + chat_result={ + "choices": [ + { + "message": { + "content": json.dumps( + { + "assertions": [ + { + "text": assertion["text"], + "reasoning": "The response matches.", + "pass": True, + } + ] + } + ) + } + } + ] + } + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-5-mini", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync("answer", {"assertions": [assertion]}) + + assert {score.name: score.score for score in result}["test_passed"] == 1.0 + assert len(client.chat_calls) == 1 + assert client.responses_calls == [] + + +def test_lm_checklist_rejects_luna_response_without_output_text(monkeypatch): + client = _RecordingClient( + responses_result={ + "id": "resp-1", + "object": "response", + "model": "gpt-56-reasoning-nano-luna", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [], + } + ], + } + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + + with pytest.raises( + ValueError, + match="Luna Responses API response contained no assistant output_text content", + ): + scorer._run_eval_sync( + "answer", + {"assertions": [{"text": "The answer is correct."}]}, + ) From 5edb9a6d77685e65211dba87f268198e74c8a264 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Wed, 7 Oct 2026 14:44:11 +0200 Subject: [PATCH 4/8] Report agent timeouts without blocking results Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evaluator/scores.py | 39 +++++++++++++++++++++ src/bcbench/commands/evaluate.py | 5 ++- src/bcbench/evaluate/base.py | 13 +++++-- src/bcbench/results/bceval_export.py | 1 + tests/test_bceval_scores.py | 51 ++++++++++++++++++++++++++++ tests/test_cli_commands.py | 33 ++++++++++++++++++ tests/test_evaluate_pipeline.py | 8 +++-- tests/test_result_writer.py | 33 ++++++++++++++++-- tests/test_review_workflows.py | 3 ++ 9 files changed, 177 insertions(+), 9 deletions(-) diff --git a/evaluator/scores.py b/evaluator/scores.py index f9ed87181..c46315063 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -7,6 +7,7 @@ _LUNA_JUDGE_MODEL = "gpt-56-reasoning-nano-luna" _LUNA_RESPONSE_FORMAT = {"type": "json_object"} +_TIMEOUT_REASON = "Agent timed out before producing output" def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -127,12 +128,43 @@ def complete(**request: object) -> dict[str, Any]: return adapted_client +def _timeout_scores(scorer: object, expected: object, metadata: dict[str, Any]) -> list[object]: + if not isinstance(expected, dict): + raise TypeError("expected must be a dict containing an 'assertions' key") + assertions = expected.get("assertions") + if not isinstance(assertions, list): + raise TypeError("expected['assertions'] must be a list") + + judged = [] + for index, assertion in enumerate(assertions): + if not isinstance(assertion, dict): + raise TypeError(f"expected['assertions'][{index}] must be a dict") + if "text" not in assertion: + raise ValueError(f"expected['assertions'][{index}] must contain a 'text' key") + judged.append({**assertion, "pass": False, "reasoning": _TIMEOUT_REASON}) + + score_metadata = {"assertions": judged, "error": _TIMEOUT_REASON} + metadata["assertionResults"] = judged + score = importlib.import_module("autoevals").Score + rate = getattr(scorer, "_rate", None) + if not callable(rate): + raise TypeError("LM Checklist scorer does not provide a callable '_rate'") + return [ + score(name="pass_rate", score=rate(judged), metadata=score_metadata), + score(name="critical_pass_rate", score=rate(judged, "critical"), metadata=score_metadata), + score(name="expected_pass_rate", score=rate(judged, "expected"), metadata=score_metadata), + score(name="aspirational_pass_rate", score=rate(judged, "aspirational"), metadata=score_metadata), + score(name="test_passed", score=0.0, metadata=score_metadata), + ] + + class LmChecklist: def __new__(cls) -> object: # bc-eval loads a custom class with this name before its built-in scorer. module = importlib.import_module("bc_eval.scorers.autoeval.lmchecklist") scorer = module.LmChecklist() request_args = scorer._request_args + run_eval_sync = scorer._run_eval_sync def responses_compatible_request_args(output: object, expected: object = None, **kwargs: object) -> dict[str, Any]: request = request_args(output, expected, **kwargs) @@ -145,7 +177,14 @@ def responses_compatible_request_args(output: object, expected: object = None, * request["client"] = _luna_responses_client(request.get("client")) return request + def timeout_aware_run_eval_sync(output: object, expected: object = None, **kwargs: object) -> object: + metadata = kwargs.get("metadata") + if isinstance(metadata, dict) and metadata.get("timeout") is True: + return _timeout_scores(scorer, expected, metadata) + return run_eval_sync(output, expected, **kwargs) + scorer._request_args = responses_compatible_request_args + scorer._run_eval_sync = timeout_aware_run_eval_sync return scorer diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index a95110104..28a917848 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -27,6 +27,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, NL2ALEntry from bcbench.evaluate import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import EvaluationOutcome from bcbench.evaluate.codereview_judge_calibration import run_calibration from bcbench.results import BaseEvaluationResult, CodeReviewResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration @@ -264,7 +265,7 @@ def evaluate_bcal( category=category, ) - category.pipeline.execute( + outcome = category.pipeline.execute( context, lambda ctx: run_bcal_agent( entry=cast(NL2ALEntry, ctx.entry), @@ -272,6 +273,8 @@ def evaluate_bcal( backend_config=backend_config, ), ) + if outcome is EvaluationOutcome.AGENT_TIMEOUT: + raise typer.Exit(code=1) logger.info("Evaluation complete!") logger.info(f"Results saved to: {run_dir}") diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index fdee6867e..11c9714ce 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -2,6 +2,7 @@ import logging from abc import ABC, abstractmethod +from enum import StrEnum from pathlib import Path from typing import Protocol @@ -14,7 +15,12 @@ logger = logging.getLogger(__name__) _config = get_config() -__all__ = ["AgentRunner", "EvaluationPipeline"] +__all__ = ["AgentRunner", "EvaluationOutcome", "EvaluationPipeline"] + + +class EvaluationOutcome(StrEnum): + COMPLETED = "completed" + AGENT_TIMEOUT = "agent_timeout" class AgentRunner[E: BaseDatasetEntry](Protocol): @@ -79,7 +85,7 @@ def execute( self, context: EvaluationContext[E], agent_runner: AgentRunner[E], - ) -> None: + ) -> EvaluationOutcome: """Template method orchestrating the evaluation flow. Executes setup, runs agent, evaluates results, and saves outcomes. @@ -99,12 +105,13 @@ def execute( result = context.category.result_class.create_agent_timeout_failure(context) self.save_result(context, result) logger.info("Agent timed out during execution, counting as failure.") - return + return EvaluationOutcome.AGENT_TIMEOUT finally: logger.info(f"Agent metrics: {context.metrics}") logger.info(f"Experiment configuration: {context.experiment}") self.evaluate(context) + return EvaluationOutcome.COMPLETED def save_result(self, context: EvaluationContext[E], result: BaseEvaluationResult) -> None: """Save result directly using result object. diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index 12a93d9d2..498fed171 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -72,6 +72,7 @@ def write_bceval_results( "run_id": run_id, "project": result.project, "error_message": result.error_message, + "timeout": result.timeout, "tool_usage": (result.metrics.tool_usage if result.metrics and result.metrics.tool_usage else None) or 0, **_experiment_metadata(result.experiment, git_ref, benchmark_version), **(completeness.to_metadata() if completeness else {}), diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py index 3a376e948..c6bd0e62d 100644 --- a/tests/test_bceval_scores.py +++ b/tests/test_bceval_scores.py @@ -38,6 +38,13 @@ def model_dump(self): return self._data +class _Score: + def __init__(self, *, name, score, metadata): + self.name = name + self.score = score + self.metadata = metadata + + def _install_lm_checklist_dependencies(monkeypatch, *, model, client): class _BuiltInLmChecklist: def __init__(self): @@ -78,6 +85,13 @@ def _run_eval_sync(self, output, expected, **kwargs): response = request_client.complete(**request) return self._process_response(response, expected["assertions"]) + @staticmethod + def _rate(assertions, level=None): + selected = [assertion for assertion in assertions if level is None or assertion.get("level") == level] + if not selected: + return 0.0 + return sum(1 for assertion in selected if assertion["pass"]) / len(selected) + def prepare_openai(*, client=None, is_async=False): assert not is_async return client or _RecordingClient() @@ -86,6 +100,11 @@ def prepare_openai(*, client=None, is_async=False): bc_eval_module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] monkeypatch.setitem(sys.modules, bc_eval_module.__name__, bc_eval_module) + autoevals_package = types.ModuleType("autoevals") + autoevals_package.__path__ = [] + autoevals_package.Score = _Score # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, autoevals_package.__name__, autoevals_package) + autoevals_module = types.ModuleType("autoevals.oai") autoevals_module.prepare_openai = prepare_openai # ty: ignore[unresolved-attribute] monkeypatch.setitem(sys.modules, autoevals_module.__name__, autoevals_module) @@ -269,3 +288,35 @@ def test_lm_checklist_rejects_luna_response_without_output_text(monkeypatch): "answer", {"assertions": [{"text": "The answer is correct."}]}, ) + + +def test_lm_checklist_scores_timeout_without_calling_judge(monkeypatch): + client = _RecordingClient() + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + metadata = {"timeout": True} + assertions = [ + {"text": "Critical behavior.", "level": "critical"}, + {"text": "Expected behavior.", "level": "expected"}, + ] + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync( + "", + {"assertions": assertions}, + metadata=metadata, + ) + + assert {score.name: score.score for score in result} == { + "pass_rate": 0.0, + "critical_pass_rate": 0.0, + "expected_pass_rate": 0.0, + "aspirational_pass_rate": 0.0, + "test_passed": 0.0, + } + assert client.responses_calls == [] + assert client.chat_calls == [] + assert metadata["assertionResults"] == [{**assertion, "pass": False, "reasoning": "Agent timed out before producing output"} for assertion in assertions] diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 34942273c..717be9a8d 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -16,6 +16,7 @@ from bcbench.commands import evaluate as evaluate_commands from bcbench.commands import run as run_commands from bcbench.dataset.dataset_entry import _BugFixTestGenBase +from bcbench.evaluate.base import EvaluationOutcome from bcbench.types import AgentMetrics, EvaluationCategory from tests.conftest import ( create_bugfix_result, @@ -125,6 +126,7 @@ def load(_dataset_path, entry_id: str): class Pipeline: def execute(self, context, _agent_runner): captured["context"] = context + return EvaluationOutcome.COMPLETED with ( patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), @@ -159,6 +161,37 @@ def test_bcal_commands_only_expose_external_command_options(command_group): assert options.isdisjoint({"--backend", "--endpoint", "--deployment"}) +def test_evaluate_bcal_exits_nonzero_after_persisted_timeout(tmp_path): + entry = create_nl2al_entry() + + class EntryClass: + @staticmethod + def load(_dataset_path, entry_id: str): + assert entry_id == entry.instance_id + return [entry] + + class Pipeline: + def execute(self, _context, _agent_runner): + return EvaluationOutcome.AGENT_TIMEOUT + + with ( + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), + patch.object(EvaluationCategory, "entry_class", new_callable=PropertyMock, return_value=EntryClass), + patch.object(EvaluationCategory, "pipeline", new_callable=PropertyMock, return_value=Pipeline()), + pytest.raises(typer.Exit) as exc_info, + ): + evaluate_commands.evaluate_bcal( + entry_id=entry.instance_id, + repo_path=tmp_path / "repo", + output_dir=tmp_path / "results", + run_id="bcal-run", + llm_command="python bridge.py", + llm_model="gpt-5.2-prod", + ) + + assert exc_info.value.exit_code == 1 + + @pytest.fixture def agent_command_category(tmp_path): entry = create_dataset_entry() diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 00cd01890..32ee7df99 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -10,7 +10,7 @@ from bcbench.commands.evaluate import MockEvaluationPipeline from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, BugFixEntry, NL2ALEntry -from bcbench.evaluate.base import AgentRunner, EvaluationPipeline +from bcbench.evaluate.base import AgentRunner, EvaluationOutcome, EvaluationPipeline from bcbench.exceptions import AgentTimeoutError from bcbench.results.base import BaseEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration @@ -64,8 +64,9 @@ def test_runs_setup_then_agent_then_evaluate(self, tmp_path): ctx = create_evaluation_context(tmp_path) pipeline = _StubPipeline[BugFixEntry]() - pipeline.execute(ctx, _noop_runner) + outcome = pipeline.execute(ctx, _noop_runner) + assert outcome is EvaluationOutcome.COMPLETED assert pipeline.setup_called assert pipeline.run_agent_called assert pipeline.evaluate_called @@ -78,8 +79,9 @@ def test_persists_timeout_result_and_skips_evaluate(self, tmp_path): timeout_config = ExperimentConfiguration(custom_instructions=True) pipeline = _StubPipeline[BugFixEntry](raise_in_run_agent=AgentTimeoutError("test timeout", metrics=timeout_metrics, config=timeout_config)) - pipeline.execute(ctx, _noop_runner) + outcome = pipeline.execute(ctx, _noop_runner) + assert outcome is EvaluationOutcome.AGENT_TIMEOUT assert pipeline.evaluate_called is False result = _read_only_result(ctx) assert result.timeout is True diff --git a/tests/test_result_writer.py b/tests/test_result_writer.py index 9dead91c3..0b5bf75e3 100644 --- a/tests/test_result_writer.py +++ b/tests/test_result_writer.py @@ -4,13 +4,42 @@ import pytest from bcbench.dataset.codereview import CodeReviewEntry -from bcbench.dataset.dataset_entry import BugFixEntry, _BugFixTestGenBase +from bcbench.dataset.dataset_entry import BugFixEntry, NL2ALEntry, _BugFixTestGenBase +from bcbench.results.base import JudgeBasedEvaluationResult from bcbench.results.bceval_export import write_bceval_results from bcbench.types import AgentHarness, AgentMetrics, EvaluationCategory, ExperimentConfiguration, PRReviewMetrics -from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result +from tests.conftest import VALID_INSTANCE_ID, create_bugfix_result, create_codereview_entry, create_codereview_result, create_nl2al_entry class TestWriteBcevalResults: + def test_exports_timeout_metadata(self, tmp_path): + entry = create_nl2al_entry() + judge_model = EvaluationCategory.NL2AL.judge_model + assert judge_model is not None + result = JudgeBasedEvaluationResult( + instance_id=entry.instance_id, + project="BaseApp", + model="gpt-55-chat-2026-04-29", + agent_name=AgentHarness.BCAL, + category=EvaluationCategory.NL2AL, + timeout=True, + error_message="Agent timed out", + judge_model=judge_model, + ) + + with patch.object(NL2ALEntry, "load", return_value=[entry]): + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="run", + output_filename="results.jsonl", + category=EvaluationCategory.NL2AL, + ) + + metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] + assert metadata["timeout"] is True + assert metadata["error_message"] == "Agent timed out" + @pytest.mark.parametrize( "metrics", [ diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index ce1096b78..98e50ef98 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -207,8 +207,11 @@ def test_agent_workflows_select_al_tool_dotnet_version_for_bc_version() -> None: def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> None: workflow = yaml.safe_load(_workflow("bcal-evaluation.yml")) + evaluation_steps = workflow["jobs"]["evaluate-with-bcal"]["steps"] summarize = workflow["jobs"]["summarize-results"] + upload = next(step for step in evaluation_steps if step["name"] == "Upload evaluation results") + assert upload["if"] == "always()" assert summarize["needs"] == ["get-entries", "evaluate-with-bcal"] assert "always()" in summarize["if"] assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" From 02b00632e042ddbfb1c5dd9807d85873b3d6d3eb Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Thu, 8 Oct 2026 13:59:12 +0200 Subject: [PATCH 5/8] Handle BCal CAPI infrastructure errors Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bcal-evaluation.yml | 1 + .github/workflows/summarize-results.yml | 11 ++- evaluator/scores.py | 21 ++++- src/bcbench/agent/bcal/agent.py | 53 ++++++++++- src/bcbench/agent/bcal/bc_eval_capi_bridge.py | 49 +++++++++- src/bcbench/commands/evaluate.py | 2 +- src/bcbench/evaluate/base.py | 15 ++- src/bcbench/exceptions.py | 20 ++++ src/bcbench/results/base.py | 26 ++++++ src/bcbench/results/bceval_export.py | 9 ++ src/bcbench/results/display.py | 2 +- tests/test_bcal_agent_provider.py | 91 ++++++++++++++++++- tests/test_bcal_capi_bridge.py | 54 +++++++++++ tests/test_bceval_scores.py | 38 +++++++- tests/test_cli_commands.py | 31 +++++++ tests/test_evaluate_pipeline.py | 30 +++++- tests/test_result_hierarchy.py | 14 ++- tests/test_result_writer.py | 36 ++++++++ tests/test_review_workflows.py | 39 ++++++-- 19 files changed, 520 insertions(+), 22 deletions(-) diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml index 486e27c91..803c2f6e6 100644 --- a/.github/workflows/bcal-evaluation.yml +++ b/.github/workflows/bcal-evaluation.yml @@ -263,4 +263,5 @@ jobs: skip-leaderboard: true expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }} artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-* + allow-unscored-results: true secrets: inherit diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index fa0eea50b..16badeab1 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -44,6 +44,11 @@ on: required: false type: string default: "*" + allow-unscored-results: + description: "Allow complete result sets containing explicit unscored infrastructure errors" + required: false + type: boolean + default: false env: BCEVAL_RESULT_FILE: bceval_results.jsonl @@ -118,6 +123,10 @@ jobs: MODEL_TAG="${MODEL_TAG//./-}" BRANCH_TAG="${{ inputs.git-ref || github.ref_name }}" BRANCH_TAG="${BRANCH_TAG//./-}" + CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}") + if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]; then + CORE_SCORE_ARGS=() + fi bceval metrics calculate \ --feature-name "BC-Bench" \ --eval-suite-name "${{ inputs.category }}" \ @@ -127,7 +136,7 @@ jobs: --input-file "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}" \ --evaluator-definitions "${{ github.workspace }}/evaluator/scores.py" \ --evaluators "${{ steps.bceval.outputs.evaluators }}" \ - --core-score "${{ steps.bceval.outputs.core_score }}" \ + "${CORE_SCORE_ARGS[@]}" \ --metric-definitions "${{ github.workspace }}/evaluator/metrics.py" \ --metrics "bc_bench_metrics" \ --use-capi ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }} diff --git a/evaluator/scores.py b/evaluator/scores.py index c46315063..7642c13d2 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -158,6 +158,19 @@ def _timeout_scores(scorer: object, expected: object, metadata: dict[str, Any]) ] +def _infrastructure_error_scores(metadata: dict[str, Any]) -> list[object]: + error = str(metadata.get("Error") or metadata.get("error_message") or "Infrastructure error") + score_metadata = {"error": error} + score = importlib.import_module("autoevals").Score + return [ + score(name="pass_rate", score=None, metadata=score_metadata), + score(name="critical_pass_rate", score=None, metadata=score_metadata), + score(name="expected_pass_rate", score=None, metadata=score_metadata), + score(name="aspirational_pass_rate", score=None, metadata=score_metadata), + score(name="test_passed", score=None, metadata=score_metadata), + ] + + class LmChecklist: def __new__(cls) -> object: # bc-eval loads a custom class with this name before its built-in scorer. @@ -179,8 +192,12 @@ def responses_compatible_request_args(output: object, expected: object = None, * def timeout_aware_run_eval_sync(output: object, expected: object = None, **kwargs: object) -> object: metadata = kwargs.get("metadata") - if isinstance(metadata, dict) and metadata.get("timeout") is True: - return _timeout_scores(scorer, expected, metadata) + if isinstance(metadata, dict): + metadata["core_score_name"] = "test_passed" + if metadata.get("infrastructure_error") is True: + return _infrastructure_error_scores(metadata) + if metadata.get("timeout") is True: + return _timeout_scores(scorer, expected, metadata) return run_eval_sync(output, expected, **kwargs) scorer._request_args = responses_compatible_request_args diff --git a/src/bcbench/agent/bcal/agent.py b/src/bcbench/agent/bcal/agent.py index 73211317d..1b920e693 100644 --- a/src/bcbench/agent/bcal/agent.py +++ b/src/bcbench/agent/bcal/agent.py @@ -1,6 +1,8 @@ """BCal agent for NL2AL evaluation — generates AL code from natural language via bcal CLI.""" +import json import logging +import os import shutil import subprocess import time @@ -11,13 +13,15 @@ from bcbench.config import get_config from bcbench.dataset import NL2ALEntry -from bcbench.exceptions import AgentError, AgentTimeoutError +from bcbench.exceptions import AgentError, AgentInfrastructureError, AgentTimeoutError from bcbench.types import AgentMetrics, ExperimentConfiguration logger = logging.getLogger(__name__) _config = get_config() _BCAL_TOOL = "bcal" +_CAPI_ERROR_FILE_ENV = "BCBENCH_CAPI_ERROR_FILE" +_CAPI_ERROR_FILENAME = ".bcbench-capi-error.json" class BCalBackendConfig(BaseModel): @@ -58,6 +62,40 @@ def _process_output(output: str | bytes | None) -> str: return (output or "").strip() +def _take_capi_infrastructure_error(error_file: Path, metrics: AgentMetrics) -> AgentInfrastructureError | None: + if not error_file.is_file(): + return None + + try: + payload = json.loads(error_file.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as error: + raise AgentError(f"Could not read BCal CAPI infrastructure error marker {error_file}: {error}") from error + finally: + error_file.unlink(missing_ok=True) + + if not isinstance(payload, dict): + raise AgentError(f"Invalid BCal CAPI infrastructure error marker: {payload!r}") + + provider = payload.get("provider") + status_code = payload.get("status_code") + message = payload.get("message") + if provider != "capi" or not isinstance(status_code, int) or not 500 <= status_code <= 599 or not isinstance(message, str): + raise AgentError(f"Invalid BCal CAPI infrastructure error marker: {payload!r}") + + return AgentInfrastructureError( + f"CAPI infrastructure error (HTTP {status_code}): {message}", + provider=provider, + status_code=status_code, + metrics=metrics, + config=ExperimentConfiguration(), + ) + + +def _raise_capi_infrastructure_error(error_file: Path, metrics: AgentMetrics) -> None: + if infrastructure_error := _take_capi_infrastructure_error(error_file, metrics): + raise infrastructure_error from None + + def _trim_prompt_echo(stdout: str, query: str) -> str: compact_query = "".join(query.split()) if not compact_query: @@ -130,6 +168,9 @@ def run_bcal_agent( export_folder = repo_path / project_name / _config.file_patterns.nl2al_export_subdir cmd_args = _bcal_cmd_args(entry, entry.get_task(), package_cache_path, export_folder, backend_config) + error_file = package_cache_path.parent / _CAPI_ERROR_FILENAME + error_file.unlink(missing_ok=True) + process_env = {**os.environ, _CAPI_ERROR_FILE_ENV: str(error_file)} logger.info(f"Export folder: {export_folder}") logger.debug(f"Package cache path: {package_cache_path}") @@ -142,18 +183,26 @@ def run_bcal_agent( cmd_args, timeout=_config.timeout.bcal_execution, check=True, + env=process_env, ) execution_time = time.monotonic() - start + metrics = AgentMetrics(execution_time=execution_time) + _raise_capi_infrastructure_error(error_file, metrics) logger.info(f"bcal CLI run complete for: {entry.instance_id}") - return AgentMetrics(execution_time=execution_time), ExperimentConfiguration() + return metrics, ExperimentConfiguration() except subprocess.TimeoutExpired: logger.exception(f"bcal CLI timed out after {_config.timeout.bcal_execution} seconds") metrics = AgentMetrics(execution_time=_config.timeout.bcal_execution) + _raise_capi_infrastructure_error(error_file, metrics) raise AgentTimeoutError("bcal CLI timed out", metrics=metrics, config=ExperimentConfiguration()) from None except subprocess.CalledProcessError as e: + metrics = AgentMetrics(execution_time=time.monotonic() - start) + _raise_capi_infrastructure_error(error_file, metrics) logger.exception(f"bcal CLI execution failed: {e.stderr}") raise AgentError(f"bcal CLI execution failed: {e}") from None + except AgentInfrastructureError: + raise except Exception: logger.exception("Unexpected error running bcal CLI") raise diff --git a/src/bcbench/agent/bcal/bc_eval_capi_bridge.py b/src/bcbench/agent/bcal/bc_eval_capi_bridge.py index bc6ee7846..589cc4d03 100644 --- a/src/bcbench/agent/bcal/bc_eval_capi_bridge.py +++ b/src/bcbench/agent/bcal/bc_eval_capi_bridge.py @@ -11,6 +11,7 @@ _CERT_FILE_ENV = "CAPI_CERT_FILE" _CERT_TENANT_ENV = "CAPI_TENANT_ID" _CERT_CLIENT_ENV = "CAPI_CLIENT_ID" +_INFRASTRUCTURE_ERROR_FILE_ENV = "BCBENCH_CAPI_ERROR_FILE" # Reasoning effort (not all models support this parameter), different models might have different available values: # Set to None to omit the parameter entirely (lets the model/service use its own default). @@ -105,6 +106,52 @@ def _maybe_install_local_cert_credential() -> None: _patch_credential_from_local_file(cert_file) +def _infrastructure_error_file() -> Path | None: + path = os.environ.get(_INFRASTRUCTURE_ERROR_FILE_ENV) + return Path(path) if path else None + + +def _clear_infrastructure_error() -> None: + error_file = _infrastructure_error_file() + if error_file is not None: + error_file.unlink(missing_ok=True) + + +def _http_status_code(error: Exception) -> int | None: + status_code = getattr(error, "status_code", None) + if not isinstance(status_code, int): + status_code = getattr(getattr(error, "response", None), "status_code", None) + return status_code if isinstance(status_code, int) else None + + +def _record_infrastructure_error(error: Exception) -> None: + status_code = _http_status_code(error) + error_file = _infrastructure_error_file() + if error_file is None or status_code is None or not 500 <= status_code <= 599: + return + + error_file.parent.mkdir(parents=True, exist_ok=True) + error_file.write_text( + json.dumps( + { + "provider": "capi", + "status_code": status_code, + "message": str(error), + } + ), + encoding="utf-8", + ) + + +def _create_completion(client: object, kwargs: dict[str, object]) -> object: + _clear_infrastructure_error() + try: + return client.chat.completions.create(**kwargs) # ty: ignore[unresolved-attribute] + except Exception as error: + _record_infrastructure_error(error) + raise + + def main() -> int: request = _load_request(sys.stdin.buffer) model = request.get("model") @@ -134,7 +181,7 @@ def main() -> int: if reasoning_effort is not None: kwargs["reasoning_effort"] = reasoning_effort - response = client.chat.completions.create(**kwargs) + response = _create_completion(client, kwargs) json.dump(_to_jsonable(response), sys.stdout) return 0 diff --git a/src/bcbench/commands/evaluate.py b/src/bcbench/commands/evaluate.py index 28a917848..db57e9fd0 100644 --- a/src/bcbench/commands/evaluate.py +++ b/src/bcbench/commands/evaluate.py @@ -273,7 +273,7 @@ def evaluate_bcal( backend_config=backend_config, ), ) - if outcome is EvaluationOutcome.AGENT_TIMEOUT: + if outcome in {EvaluationOutcome.AGENT_INFRASTRUCTURE_ERROR, EvaluationOutcome.AGENT_TIMEOUT}: raise typer.Exit(code=1) logger.info("Evaluation complete!") diff --git a/src/bcbench/evaluate/base.py b/src/bcbench/evaluate/base.py index 11c9714ce..b62b4416c 100644 --- a/src/bcbench/evaluate/base.py +++ b/src/bcbench/evaluate/base.py @@ -8,7 +8,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry -from bcbench.exceptions import AgentTimeoutError +from bcbench.exceptions import AgentInfrastructureError, AgentTimeoutError from bcbench.results import BaseEvaluationResult from bcbench.types import AgentMetrics, EvaluationContext, ExperimentConfiguration @@ -20,6 +20,7 @@ class EvaluationOutcome(StrEnum): COMPLETED = "completed" + AGENT_INFRASTRUCTURE_ERROR = "agent_infrastructure_error" AGENT_TIMEOUT = "agent_timeout" @@ -106,6 +107,18 @@ def execute( self.save_result(context, result) logger.info("Agent timed out during execution, counting as failure.") return EvaluationOutcome.AGENT_TIMEOUT + except AgentInfrastructureError as e: + context.metrics = e.metrics + context.experiment = e.config + result = context.category.result_class.create_agent_infrastructure_failure( + context, + error_message=str(e), + provider=e.provider, + status_code=e.status_code, + ) + self.save_result(context, result) + logger.warning("Agent execution stopped because an infrastructure dependency failed.") + return EvaluationOutcome.AGENT_INFRASTRUCTURE_ERROR finally: logger.info(f"Agent metrics: {context.metrics}") logger.info(f"Experiment configuration: {context.experiment}") diff --git a/src/bcbench/exceptions.py b/src/bcbench/exceptions.py index f08a18b17..3ed947f61 100644 --- a/src/bcbench/exceptions.py +++ b/src/bcbench/exceptions.py @@ -9,6 +9,7 @@ __all__ = [ "AgentError", + "AgentInfrastructureError", "BCBenchError", "CollectionError", "ConfigurationError", @@ -90,6 +91,25 @@ class AgentError(BCBenchError): """Agent execution errors.""" +class AgentInfrastructureError(AgentError): + """A dependency failure that prevented the agent from producing an evaluable result.""" + + def __init__( + self, + message: str, + *, + provider: str, + status_code: int | None = None, + metrics: AgentMetrics | None = None, + config: ExperimentConfiguration | None = None, + ) -> None: + self.provider = provider + self.status_code = status_code + self.metrics = metrics + self.config = config + super().__init__(message) + + class AgentTimeoutError(BCBenchError): """Agent execution timeout errors.""" diff --git a/src/bcbench/results/base.py b/src/bcbench/results/base.py index 809486724..4e68d1a62 100644 --- a/src/bcbench/results/base.py +++ b/src/bcbench/results/base.py @@ -24,6 +24,9 @@ class BaseEvaluationResult(BaseModel): agent_version: str | None = None timeout: bool = False + infrastructure_error: bool = False + error_provider: str | None = None + error_status_code: int | None = None output: str = "" error_message: str | None = None @@ -56,6 +59,23 @@ def _base_fields[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> dic def create_agent_timeout_failure[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) -> Self: return cls(**cls._base_fields(context), timeout=True, error_message="Agent timed out") + @classmethod + def create_agent_infrastructure_failure[E: BaseDatasetEntry]( + cls, + context: EvaluationContext[E], + *, + error_message: str, + provider: str, + status_code: int | None, + ) -> Self: + return cls( + **cls._base_fields(context), + infrastructure_error=True, + error_provider=provider, + error_status_code=status_code, + error_message=error_message, + ) + def save(self, output_dir: Path, result_file: str) -> None: output_file = output_dir / result_file output_dir.mkdir(parents=True, exist_ok=True) @@ -71,6 +91,8 @@ def status_label(self) -> str: """Short human-readable label for the result status shown in tables (e.g. 'Completed', 'Timeout').""" if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" if self.error_message: return "Error" return "Completed" @@ -131,6 +153,8 @@ def create_result[E: BaseDatasetEntry](cls, context: EvaluationContext[E], outpu def status_label(self) -> str: if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" return "Success" if self.resolved else "Failed" @property @@ -195,6 +219,8 @@ def create_empty_output[E: BaseDatasetEntry](cls, context: EvaluationContext[E]) def status_label(self) -> str: if self.timeout: return "Timeout" + if self.infrastructure_error: + return "Infrastructure Error" if self.error_message: return "Error" return "Unscored" diff --git a/src/bcbench/results/bceval_export.py b/src/bcbench/results/bceval_export.py index 498fed171..091e5e241 100644 --- a/src/bcbench/results/bceval_export.py +++ b/src/bcbench/results/bceval_export.py @@ -77,6 +77,15 @@ def write_bceval_results( **_experiment_metadata(result.experiment, git_ref, benchmark_version), **(completeness.to_metadata() if completeness else {}), } + if result.infrastructure_error: + metadata.update( + { + "infrastructure_error": True, + "error_provider": result.error_provider, + "error_status_code": result.error_status_code, + "Error": result.error_message, + } + ) bceval_result = { "id": result.instance_id, diff --git a/src/bcbench/results/display.py b/src/bcbench/results/display.py index a9d2e9cde..3acd07c03 100644 --- a/src/bcbench/results/display.py +++ b/src/bcbench/results/display.py @@ -16,7 +16,7 @@ def _status_style(status_label: str) -> tuple[str, str]: """Return (rich_color, github_emoji) for a status label.""" - if status_label in ("Timeout", "Error", "Failed"): + if status_label in ("Timeout", "Infrastructure Error", "Error", "Failed"): return "red", ":x:" if status_label == "Unscored": return "yellow", ":grey_question:" diff --git a/tests/test_bcal_agent_provider.py b/tests/test_bcal_agent_provider.py index f415369d5..80819034c 100644 --- a/tests/test_bcal_agent_provider.py +++ b/tests/test_bcal_agent_provider.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess from pathlib import Path from unittest.mock import MagicMock, patch @@ -10,7 +11,7 @@ from bcbench.agent.bcal import BCalBackendConfig from bcbench.agent.bcal import agent as bcal_agent -from bcbench.exceptions import AgentError, AgentTimeoutError +from bcbench.exceptions import AgentError, AgentInfrastructureError, AgentTimeoutError from tests.conftest import create_nl2al_entry @@ -113,6 +114,94 @@ def fake_run(args: list[str], **_: object) -> MagicMock: assert "--llm-command=python bridge.py" in captured["args"] assert not any(a.startswith("--deployment=") for a in captured["args"]) + def test_capi_5xx_marker_raises_typed_infrastructure_error(self, workspace: Path): + entry = create_nl2al_entry() + + def fake_run(_args: list[str], **kwargs: object) -> None: + process_env = kwargs["env"] + assert isinstance(process_env, dict) + marker = Path(process_env[bcal_agent._CAPI_ERROR_FILE_ENV]) + marker.write_text( + json.dumps( + { + "provider": "capi", + "status_code": 500, + "message": "DependencyFailure: InternalServerError", + } + ), + encoding="utf-8", + ) + raise subprocess.CalledProcessError(returncode=1, cmd=["bcal"]) + + with ( + patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), + patch.object(subprocess, "run", side_effect=fake_run), + pytest.raises(AgentInfrastructureError, match="HTTP 500") as exc_info, + ): + bcal_agent.run_bcal_agent( + entry=entry, + repo_path=workspace, + backend_config=BCalBackendConfig( + command="python bridge.py", + model="gpt-5", + ), + ) + + assert exc_info.value.provider == "capi" + assert exc_info.value.status_code == 500 + assert exc_info.value.metrics is not None + assert not (workspace / "JobBudgetVsActualReport" / bcal_agent._CAPI_ERROR_FILENAME).exists() + + def test_capi_5xx_marker_takes_precedence_over_process_timeout(self, workspace: Path): + entry = create_nl2al_entry() + + def fake_run(_args: list[str], **kwargs: object) -> None: + process_env = kwargs["env"] + assert isinstance(process_env, dict) + Path(process_env[bcal_agent._CAPI_ERROR_FILE_ENV]).write_text( + json.dumps( + { + "provider": "capi", + "status_code": 503, + "message": "Service unavailable", + } + ), + encoding="utf-8", + ) + raise subprocess.TimeoutExpired(cmd=["bcal"], timeout=1) + + with ( + patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), + patch.object(subprocess, "run", side_effect=fake_run), + pytest.raises(AgentInfrastructureError, match="HTTP 503"), + ): + bcal_agent.run_bcal_agent( + entry=entry, + repo_path=workspace, + backend_config=BCalBackendConfig( + command="python bridge.py", + ), + ) + + def test_generic_nonzero_exit_remains_agent_error(self, workspace: Path): + entry = create_nl2al_entry() + failure = subprocess.CalledProcessError(returncode=2, cmd=["bcal"]) + + with ( + patch.object(bcal_agent, "_resolve_bcal_executable", return_value="C:\\fake\\bcal.exe"), + patch.object(subprocess, "run", side_effect=failure), + pytest.raises(AgentError, match="status 2") as exc_info, + ): + bcal_agent.run_bcal_agent( + entry=entry, + repo_path=workspace, + backend_config=BCalBackendConfig( + command="python bridge.py", + ), + ) + + assert not isinstance(exc_info.value, AgentInfrastructureError) + class TestRunBcalPrompt: @pytest.mark.parametrize( diff --git a/tests/test_bcal_capi_bridge.py b/tests/test_bcal_capi_bridge.py index a4f63b487..c7825c632 100644 --- a/tests/test_bcal_capi_bridge.py +++ b/tests/test_bcal_capi_bridge.py @@ -10,6 +10,24 @@ from bcbench.agent.bcal import bc_eval_capi_bridge +class _CapiClient: + def __init__(self, result: object = None, error: Exception | None = None) -> None: + self.result = result + self.error = error + self.chat = types.SimpleNamespace(completions=types.SimpleNamespace(create=self.create)) + + def create(self, **_kwargs: object) -> object: + if self.error is not None: + raise self.error + return self.result + + +class _HttpError(Exception): + def __init__(self, status_code: int) -> None: + self.status_code = status_code + super().__init__(f"HTTP {status_code}") + + def test_load_request_accepts_utf8_bom_from_bcal_windows_stdin(): request = { "model": "gpt-5", @@ -21,6 +39,42 @@ def test_load_request_accepts_utf8_bom_from_bcal_windows_stdin(): assert bc_eval_capi_bridge._load_request(input_stream) == request +@pytest.mark.parametrize("status_code", [500, 503, 599]) +def test_create_completion_records_capi_5xx_as_infrastructure_error(monkeypatch, tmp_path, status_code): + error_file = tmp_path / "capi-error.json" + monkeypatch.setenv(bc_eval_capi_bridge._INFRASTRUCTURE_ERROR_FILE_ENV, str(error_file)) + + with pytest.raises(_HttpError, match=f"HTTP {status_code}"): + bc_eval_capi_bridge._create_completion(_CapiClient(error=_HttpError(status_code)), {}) + + assert json.loads(error_file.read_text(encoding="utf-8")) == { + "provider": "capi", + "status_code": status_code, + "message": f"HTTP {status_code}", + } + + +@pytest.mark.parametrize("status_code", [400, 401, 429]) +def test_create_completion_does_not_classify_non_5xx_failures(monkeypatch, tmp_path, status_code): + error_file = tmp_path / "capi-error.json" + monkeypatch.setenv(bc_eval_capi_bridge._INFRASTRUCTURE_ERROR_FILE_ENV, str(error_file)) + + with pytest.raises(_HttpError, match=f"HTTP {status_code}"): + bc_eval_capi_bridge._create_completion(_CapiClient(error=_HttpError(status_code)), {}) + + assert not error_file.exists() + + +def test_create_completion_clears_stale_infrastructure_error_after_success(monkeypatch, tmp_path): + error_file = tmp_path / "capi-error.json" + error_file.write_text("stale", encoding="utf-8") + monkeypatch.setenv(bc_eval_capi_bridge._INFRASTRUCTURE_ERROR_FILE_ENV, str(error_file)) + expected = {"choices": []} + + assert bc_eval_capi_bridge._create_completion(_CapiClient(result=expected), {}) == expected + assert not error_file.exists() + + @pytest.fixture def fake_capi_auth(monkeypatch): """Provide stubs for bc_eval.capi.capi_auth and azure.identity used by the cert patcher.""" diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py index c6bd0e62d..a4202c1c5 100644 --- a/tests/test_bceval_scores.py +++ b/tests/test_bceval_scores.py @@ -250,9 +250,11 @@ def test_lm_checklist_delegates_non_luna_requests(monkeypatch): ) scorer = cast(built_in, scores.LmChecklist()) - result = scorer._run_eval_sync("answer", {"assertions": [assertion]}) + metadata = {} + result = scorer._run_eval_sync("answer", {"assertions": [assertion]}, metadata=metadata) assert {score.name: score.score for score in result}["test_passed"] == 1.0 + assert metadata["core_score_name"] == "test_passed" assert len(client.chat_calls) == 1 assert client.responses_calls == [] @@ -319,4 +321,38 @@ def test_lm_checklist_scores_timeout_without_calling_judge(monkeypatch): } assert client.responses_calls == [] assert client.chat_calls == [] + assert metadata["core_score_name"] == "test_passed" assert metadata["assertionResults"] == [{**assertion, "pass": False, "reasoning": "Agent timed out before producing output"} for assertion in assertions] + + +def test_lm_checklist_skips_infrastructure_error_without_calling_judge(monkeypatch): + client = _RecordingClient() + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + error_message = "CAPI infrastructure error (HTTP 500): DependencyFailure" + metadata = { + "infrastructure_error": True, + "Error": error_message, + } + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync( + "", + {"assertions": [{"text": "Critical behavior.", "level": "critical"}]}, + metadata=metadata, + ) + + assert {score.name: score.score for score in result} == { + "pass_rate": None, + "critical_pass_rate": None, + "expected_pass_rate": None, + "aspirational_pass_rate": None, + "test_passed": None, + } + assert all(score.metadata == {"error": error_message} for score in result) + assert metadata["core_score_name"] == "test_passed" + assert client.responses_calls == [] + assert client.chat_calls == [] diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 717be9a8d..c019c69ab 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -192,6 +192,37 @@ def execute(self, _context, _agent_runner): assert exc_info.value.exit_code == 1 +def test_evaluate_bcal_exits_nonzero_after_persisted_infrastructure_error(tmp_path): + entry = create_nl2al_entry() + + class EntryClass: + @staticmethod + def load(_dataset_path, entry_id: str): + assert entry_id == entry.instance_id + return [entry] + + class Pipeline: + def execute(self, _context, _agent_runner): + return EvaluationOutcome.AGENT_INFRASTRUCTURE_ERROR + + with ( + patch.object(EvaluationCategory, "dataset_path", new_callable=PropertyMock, return_value=tmp_path / "nl2al.jsonl"), + patch.object(EvaluationCategory, "entry_class", new_callable=PropertyMock, return_value=EntryClass), + patch.object(EvaluationCategory, "pipeline", new_callable=PropertyMock, return_value=Pipeline()), + pytest.raises(typer.Exit) as exc_info, + ): + evaluate_commands.evaluate_bcal( + entry_id=entry.instance_id, + repo_path=tmp_path / "repo", + output_dir=tmp_path / "results", + run_id="bcal-run", + llm_command="python bridge.py", + llm_model="gpt-5.2-prod", + ) + + assert exc_info.value.exit_code == 1 + + @pytest.fixture def agent_command_category(tmp_path): entry = create_dataset_entry() diff --git a/tests/test_evaluate_pipeline.py b/tests/test_evaluate_pipeline.py index 32ee7df99..3af9de93b 100644 --- a/tests/test_evaluate_pipeline.py +++ b/tests/test_evaluate_pipeline.py @@ -11,7 +11,7 @@ from bcbench.config import get_config from bcbench.dataset import BaseDatasetEntry, BugFixEntry, NL2ALEntry from bcbench.evaluate.base import AgentRunner, EvaluationOutcome, EvaluationPipeline -from bcbench.exceptions import AgentTimeoutError +from bcbench.exceptions import AgentInfrastructureError, AgentTimeoutError from bcbench.results.base import BaseEvaluationResult, JudgeBasedEvaluationResult from bcbench.types import AgentMetrics, EvaluationCategory, EvaluationContext, ExperimentConfiguration from tests.conftest import create_codereview_entry, create_dataset_entry, create_evaluation_context, create_ext_advisor_entry, create_ext_implement_entry, create_nl2al_entry @@ -105,6 +105,34 @@ def test_persists_category_specific_timeout_result(self, tmp_path): assert result.judge_model == EvaluationCategory.NL2AL.judge_model +class TestExecuteAgentInfrastructureError: + def test_persists_infrastructure_result_and_skips_evaluate(self, tmp_path): + entry = create_nl2al_entry() + ctx = create_evaluation_context(tmp_path, entry=entry, category=EvaluationCategory.NL2AL) + metrics = AgentMetrics(execution_time=42.0) + error = AgentInfrastructureError( + "CAPI infrastructure error (HTTP 500): DependencyFailure", + provider="capi", + status_code=500, + metrics=metrics, + config=ExperimentConfiguration(), + ) + pipeline = _StubPipeline[NL2ALEntry](raise_in_run_agent=error) + + outcome = pipeline.execute(ctx, lambda _: (None, None)) + + assert outcome is EvaluationOutcome.AGENT_INFRASTRUCTURE_ERROR + assert pipeline.evaluate_called is False + result = _read_only_result(ctx) + assert isinstance(result, JudgeBasedEvaluationResult) + assert result.infrastructure_error is True + assert result.error_provider == "capi" + assert result.error_status_code == 500 + assert result.error_message == str(error) + assert result.metrics == metrics + assert result.status_label == "Infrastructure Error" + + class TestExecuteEvaluateError: def test_unexpected_exceptions_still_propagate(self, tmp_path): ctx = create_evaluation_context(tmp_path) diff --git a/tests/test_result_hierarchy.py b/tests/test_result_hierarchy.py index 556888b7e..5bec7d2cb 100644 --- a/tests/test_result_hierarchy.py +++ b/tests/test_result_hierarchy.py @@ -19,7 +19,7 @@ from bcbench.results.base import BaseEvaluationResult, ExecutionBasedEvaluationResult, JudgeBasedEvaluationResult from bcbench.results.bugfix import BugFixResult -from bcbench.results.display import create_console_summary, create_github_job_summary +from bcbench.results.display import _status_style, create_console_summary, create_github_job_summary from bcbench.results.summary import ( EvaluationResultSummary, ExecutionBasedEvaluationResultSummary, @@ -87,6 +87,18 @@ def test_execution_based_failed(self): result = create_bugfix_result(resolved=False, build=True, error_message="Tests failed") assert result.status_label == "Failed" + def test_infrastructure_error_is_distinct_and_displayed_as_an_error(self, tmp_path): + ctx = create_evaluation_context(tmp_path, entry=create_nl2al_entry(), category=EvaluationCategory.NL2AL) + result = JudgeBasedEvaluationResult.create_agent_infrastructure_failure( + ctx, + error_message="CAPI HTTP 500", + provider="capi", + status_code=500, + ) + + assert result.status_label == "Infrastructure Error" + assert _status_style(result.status_label) == ("red", ":x:") + # --------------------------------------------------------------------------- # category_metrics diff --git a/tests/test_result_writer.py b/tests/test_result_writer.py index 0b5bf75e3..9a2e36d6a 100644 --- a/tests/test_result_writer.py +++ b/tests/test_result_writer.py @@ -40,6 +40,40 @@ def test_exports_timeout_metadata(self, tmp_path): assert metadata["timeout"] is True assert metadata["error_message"] == "Agent timed out" + def test_exports_infrastructure_error_without_model_score(self, tmp_path): + entry = create_nl2al_entry() + judge_model = EvaluationCategory.NL2AL.judge_model + assert judge_model is not None + error_message = "CAPI infrastructure error (HTTP 500): DependencyFailure" + result = JudgeBasedEvaluationResult( + instance_id=entry.instance_id, + project="BaseApp", + model="gpt-55-chat-2026-04-29", + agent_name=AgentHarness.BCAL, + category=EvaluationCategory.NL2AL, + infrastructure_error=True, + error_provider="capi", + error_status_code=500, + error_message=error_message, + judge_model=judge_model, + ) + + with patch.object(NL2ALEntry, "load", return_value=[entry]): + write_bceval_results( + results=[result], + out_dir=tmp_path, + run_id="run", + output_filename="results.jsonl", + category=EvaluationCategory.NL2AL, + ) + + metadata = json.loads((tmp_path / "results.jsonl").read_text(encoding="utf-8"))["metadata"] + assert metadata["infrastructure_error"] is True + assert metadata["error_provider"] == "capi" + assert metadata["error_status_code"] == 500 + assert metadata["Error"] == error_message + assert "test_passed" not in metadata + @pytest.mark.parametrize( "metrics", [ @@ -99,6 +133,8 @@ def test_writes_bceval_results_with_all_fields(self, tmp_path, sample_dataset_fi assert data["metadata"]["agent_version"] == "1.2.3" # bug-fix is not judge-scored, so no judge model is exported assert "judge_model" not in data["metadata"] + assert "infrastructure_error" not in data["metadata"] + assert "Error" not in data["metadata"] assert data["metadata"]["prompt_tokens"] == 5000 assert data["metadata"]["completion_tokens"] == 1200 assert data["metadata"]["latency"] == 120.5 diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 98e50ef98..6bed80875 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -217,6 +217,7 @@ def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" assert summarize["with"]["results-dir"] == "evaluation_results" assert summarize["with"]["artifact-pattern"] == "evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*" + assert summarize["with"]["allow-unscored-results"] is True def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: @@ -234,12 +235,32 @@ def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> None: - workflow = _workflow("summarize-results.yml") - - assert "continue-on-error: ${{ inputs.expected-total >= 0 }}" in workflow - assert "evaluation_completeness.json" in workflow - assert "bceval_results.jsonl" in workflow - assert "steps.summarize.outputs.complete == 'true'" in workflow - assert "bcbench.bceval_runner" not in workflow - assert "if: always()" in workflow - assert "bcbench result require-complete" in workflow + workflow_text = _workflow("summarize-results.yml") + workflow = yaml.safe_load(workflow_text) + + assert "continue-on-error: ${{ inputs.expected-total >= 0 }}" in workflow_text + assert "evaluation_completeness.json" in workflow_text + assert "bceval_results.jsonl" in workflow_text + assert "steps.summarize.outputs.complete == 'true'" in workflow_text + assert "bcbench.bceval_runner" not in workflow_text + assert "if: always()" in workflow_text + assert "bcbench result require-complete" in workflow_text + assert workflow[True]["workflow_call"]["inputs"]["allow-unscored-results"]["default"] is False + assert 'CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")' in workflow_text + assert 'if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]' in workflow_text + assert '"${CORE_SCORE_ARGS[@]}"' in workflow_text + + +def test_only_bcal_summary_allows_unscored_infrastructure_results() -> None: + for workflow_name in ("copilot-evaluation.yml", "claude-evaluation.yml", "pr-review-evaluation.yml", "CI.yml"): + workflow = yaml.safe_load(_workflow(workflow_name)) + summarize = workflow["jobs"]["summarize-results"]["with"] + assert "allow-unscored-results" not in summarize + + +def test_bcal_keeps_bceval_0_3_14_pinned_for_bridge_and_scoring() -> None: + bcal_workflow = _workflow("bcal-evaluation.yml") + summarize_workflow = _workflow("summarize-results.yml") + + assert bcal_workflow.count('"bc-eval[capi]==0.3.14"') == 1 + assert summarize_workflow.count("bc-eval[capi]==0.3.14") == 1 From 1bbe3e76d44bbf44cdd6923f6d69899a0d4f68a2 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Thu, 8 Oct 2026 14:12:48 +0200 Subject: [PATCH 6/8] Adapt temporary BCal fixes to latest main Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/bcbench/agent/bcal/agent.py | 2 +- tests/test_review_workflows.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bcbench/agent/bcal/agent.py b/src/bcbench/agent/bcal/agent.py index 1b920e693..89eb5c2df 100644 --- a/src/bcbench/agent/bcal/agent.py +++ b/src/bcbench/agent/bcal/agent.py @@ -177,8 +177,8 @@ def run_bcal_agent( logger.debug(f"Using prompt:\n{entry.get_task()}") logger.debug(f"bcal CLI command: {cmd_args}") + start = time.monotonic() try: - start = time.monotonic() subprocess.run( cmd_args, timeout=_config.timeout.bcal_execution, diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 6bed80875..cc7aff719 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -215,7 +215,7 @@ def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> assert summarize["needs"] == ["get-entries", "evaluate-with-bcal"] assert "always()" in summarize["if"] assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" - assert summarize["with"]["results-dir"] == "evaluation_results" + assert summarize["with"]["results-dir"] == "${{ needs.evaluate-with-bcal.outputs.results-dir }}" assert summarize["with"]["artifact-pattern"] == "evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*" assert summarize["with"]["allow-unscored-results"] is True From 65c56878fb15916876b165f266e7822aef1b29bd Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Fri, 9 Oct 2026 11:09:59 +0200 Subject: [PATCH 7/8] Add BCal artifact recovery mode Allow one failed NL2AL shard to be rerun while reusing a prior attempt's complete artifacts for final completeness checks and scoring. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/bcal-evaluation.yml | 77 ++++++++++++++++++++++--- .github/workflows/summarize-results.yml | 22 +++++++ tests/test_review_workflows.py | 45 ++++++++++++++- 3 files changed, 135 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml index 803c2f6e6..655cdba79 100644 --- a/.github/workflows/bcal-evaluation.yml +++ b/.github/workflows/bcal-evaluation.yml @@ -21,6 +21,21 @@ on: required: false default: true type: boolean + recovery-source-run-id: + description: "Prior workflow run ID whose artifacts should be combined with one recovered entry" + required: false + default: "" + type: string + recovery-source-run-attempt: + description: "Prior workflow run attempt containing the reusable artifacts" + required: false + default: "" + type: string + recovery-entry: + description: "Single NL2AL entry to rerun while reusing artifacts from the prior attempt" + required: false + default: "" + type: string schedule: - cron: "0 3 * * 0" @@ -37,13 +52,56 @@ jobs: get-entries: uses: $/.github/workflows/get-entries.yml with: - test-run: ${{ inputs.test-run || false }} + test-run: ${{ inputs.recovery-entry == '' && inputs.test-run }} category: nl2al + select-entries: + runs-on: ubuntu-latest + needs: get-entries + outputs: + entries: ${{ steps.select.outputs.entries }} + steps: + - name: Select entries for evaluation + id: select + env: + ALL_ENTRIES: ${{ needs.get-entries.outputs.entries }} + RECOVERY_SOURCE_RUN_ID: ${{ inputs.recovery-source-run-id }} + RECOVERY_SOURCE_RUN_ATTEMPT: ${{ inputs.recovery-source-run-attempt }} + RECOVERY_ENTRY: ${{ inputs.recovery-entry }} + TEST_RUN: ${{ inputs.test-run }} + run: | + if [[ -z "$RECOVERY_SOURCE_RUN_ID" && -z "$RECOVERY_SOURCE_RUN_ATTEMPT" && -z "$RECOVERY_ENTRY" ]]; then + echo "entries=$ALL_ENTRIES" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [[ -z "$RECOVERY_SOURCE_RUN_ID" || -z "$RECOVERY_SOURCE_RUN_ATTEMPT" || -z "$RECOVERY_ENTRY" ]]; then + echo "::error::recovery-source-run-id, recovery-source-run-attempt, and recovery-entry must be provided together." + exit 1 + fi + if [[ "$TEST_RUN" == "true" ]]; then + echo "::error::Recovery runs must set test-run to false so the complete result is scored and uploaded." + exit 1 + fi + if ! [[ "$RECOVERY_SOURCE_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::recovery-source-run-id must be a positive integer." + exit 1 + fi + if ! [[ "$RECOVERY_SOURCE_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::recovery-source-run-attempt must be a positive integer." + exit 1 + fi + if ! jq -e --arg entry "$RECOVERY_ENTRY" 'index($entry) != null' <<< "$ALL_ENTRIES" > /dev/null; then + echo "::error::Recovery entry '$RECOVERY_ENTRY' is not in the full NL2AL dataset." + exit 1 + fi + + echo "entries=$(jq -cn --arg entry "$RECOVERY_ENTRY" '[$entry]')" >> "$GITHUB_OUTPUT" + prepare-bccontainerhelper: runs-on: windows-latest - needs: get-entries - if: needs.get-entries.outputs.entries != '[]' + needs: select-entries + if: needs.select-entries.outputs.entries != '[]' steps: - name: Cache pinned BcContainerHelper id: cache-bccontainerhelper @@ -77,10 +135,10 @@ jobs: evaluate-with-bcal: runs-on: windows-latest - needs: [get-entries, prepare-bccontainerhelper] + needs: [select-entries, prepare-bccontainerhelper] outputs: results-dir: ${{ env.EVALUATION_RESULTS_DIR }} - if: needs.get-entries.outputs.entries != '[]' + if: needs.select-entries.outputs.entries != '[]' environment: name: ado-read deployment: false @@ -92,7 +150,7 @@ jobs: fail-fast: false max-parallel: 32 matrix: - entry: ${{ fromJson(needs.get-entries.outputs.entries) }} + entry: ${{ fromJson(needs.select-entries.outputs.entries) }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -248,10 +306,11 @@ jobs: retention-days: ${{ inputs.test-run && 1 || 30 }} summarize-results: - needs: [get-entries, evaluate-with-bcal] - if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }} + needs: [get-entries, select-entries, evaluate-with-bcal] + if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.select-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }} uses: $/.github/workflows/summarize-results.yml permissions: + actions: read contents: write id-token: write with: @@ -263,5 +322,7 @@ jobs: skip-leaderboard: true expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }} artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-* + recovery-artifact-run-id: ${{ inputs.recovery-source-run-id }} + recovery-artifact-pattern: ${{ inputs.recovery-source-run-id != '' && format('evaluation-results-{0}-{1}-*', inputs.recovery-source-run-id, inputs.recovery-source-run-attempt) || '' }} allow-unscored-results: true secrets: inherit diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index 16badeab1..c636c6964 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -44,6 +44,16 @@ on: required: false type: string default: "*" + recovery-artifact-run-id: + description: "Prior workflow run ID containing artifacts to merge into this run" + required: false + type: string + default: "" + recovery-artifact-pattern: + description: "Artifact glob to download from the recovery source run" + required: false + type: string + default: "" allow-unscored-results: description: "Allow complete result sets containing explicit unscored infrastructure errors" required: false @@ -62,6 +72,7 @@ jobs: name: ado-read deployment: false permissions: + actions: read contents: write id-token: write runs-on: ubuntu-latest @@ -80,6 +91,17 @@ jobs: pattern: ${{ inputs.artifact-pattern }} merge-multiple: true + - name: Download recovery evaluation results + if: inputs.recovery-artifact-run-id != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ inputs.recovery-artifact-run-id }} + path: ${{ inputs.results-dir }}/${{ github.run_id }}/recovered/ + pattern: ${{ inputs.recovery-artifact-pattern }} + merge-multiple: true + - name: Summarize evaluation results id: summarize run: >- diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index cc7aff719..78c0fccd8 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -212,7 +212,7 @@ def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> upload = next(step for step in evaluation_steps if step["name"] == "Upload evaluation results") assert upload["if"] == "always()" - assert summarize["needs"] == ["get-entries", "evaluate-with-bcal"] + assert summarize["needs"] == ["get-entries", "select-entries", "evaluate-with-bcal"] assert "always()" in summarize["if"] assert summarize["with"]["expected-total"] == "${{ fromJSON(needs.get-entries.outputs.entry-count) }}" assert summarize["with"]["results-dir"] == "${{ needs.evaluate-with-bcal.outputs.results-dir }}" @@ -220,6 +220,34 @@ def test_bcal_summary_runs_after_matrix_failures_and_enforces_completeness() -> assert summarize["with"]["allow-unscored-results"] is True +def test_bcal_recovery_reruns_one_entry_and_merges_prior_attempt_artifacts() -> None: + workflow = yaml.safe_load(_workflow("bcal-evaluation.yml")) + dispatch_inputs = workflow[True]["workflow_dispatch"]["inputs"] + select = workflow["jobs"]["select-entries"] + selection_step = select["steps"][0] + evaluate = workflow["jobs"]["evaluate-with-bcal"] + summarize = workflow["jobs"]["summarize-results"] + + for input_name in ("recovery-source-run-id", "recovery-source-run-attempt", "recovery-entry"): + assert dispatch_inputs[input_name]["default"] == "" + assert dispatch_inputs[input_name]["type"] == "string" + + assert workflow["jobs"]["get-entries"]["with"]["test-run"] == "${{ inputs.recovery-entry == '' && inputs.test-run }}" + assert select["needs"] == "get-entries" + assert selection_step["env"]["ALL_ENTRIES"] == "${{ needs.get-entries.outputs.entries }}" + assert "must be provided together" in selection_step["run"] + assert "Recovery runs must set test-run to false" in selection_step["run"] + assert "'index($entry) != null'" in selection_step["run"] + assert "jq -cn --arg entry" in selection_step["run"] + assert evaluate["needs"] == ["select-entries", "prepare-bccontainerhelper"] + assert evaluate["strategy"]["matrix"]["entry"] == "${{ fromJson(needs.select-entries.outputs.entries) }}" + assert summarize["permissions"]["actions"] == "read" + assert summarize["with"]["recovery-artifact-run-id"] == "${{ inputs.recovery-source-run-id }}" + assert summarize["with"]["recovery-artifact-pattern"] == ( + "${{ inputs.recovery-source-run-id != '' && format('evaluation-results-{0}-{1}-*', inputs.recovery-source-run-id, inputs.recovery-source-run-attempt) || '' }}" + ) + + def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: workflow_text = _workflow("bcal-evaluation.yml") workflow = yaml.safe_load(workflow_text) @@ -237,6 +265,10 @@ def test_bcal_downloads_one_pinned_cached_bccontainerhelper() -> None: def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> None: workflow_text = _workflow("summarize-results.yml") workflow = yaml.safe_load(workflow_text) + workflow_inputs = workflow[True]["workflow_call"]["inputs"] + steps = workflow["jobs"]["summarize-results"]["steps"] + current_download = next(step for step in steps if step["name"] == "Download all evaluation results") + recovery_download = next(step for step in steps if step["name"] == "Download recovery evaluation results") assert "continue-on-error: ${{ inputs.expected-total >= 0 }}" in workflow_text assert "evaluation_completeness.json" in workflow_text @@ -249,6 +281,17 @@ def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> assert 'CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")' in workflow_text assert 'if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]' in workflow_text assert '"${CORE_SCORE_ARGS[@]}"' in workflow_text + assert workflow["jobs"]["summarize-results"]["permissions"]["actions"] == "read" + assert workflow_inputs["recovery-artifact-run-id"]["default"] == "" + assert workflow_inputs["recovery-artifact-pattern"]["default"] == "" + assert steps.index(current_download) < steps.index(recovery_download) + assert recovery_download["if"] == "inputs.recovery-artifact-run-id != ''" + assert recovery_download["with"]["github-token"] == "${{ github.token }}" + assert recovery_download["with"]["repository"] == "${{ github.repository }}" + assert recovery_download["with"]["run-id"] == "${{ inputs.recovery-artifact-run-id }}" + assert recovery_download["with"]["path"] == "${{ inputs.results-dir }}/${{ github.run_id }}/recovered/" + assert recovery_download["with"]["pattern"] == "${{ inputs.recovery-artifact-pattern }}" + assert recovery_download["with"]["merge-multiple"] is True def test_only_bcal_summary_allows_unscored_infrastructure_results() -> None: From eb06f2b332534c328aa0151a3e485695f9d9f718 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Fri, 9 Oct 2026 11:14:09 +0200 Subject: [PATCH 8/8] Inherit reusable summary permissions Let each workflow caller grant only the permissions it needs so BCal recovery can read prior artifacts without invalidating normal callers. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/summarize-results.yml | 4 ---- tests/test_review_workflows.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml index c636c6964..c7e0c0eba 100644 --- a/.github/workflows/summarize-results.yml +++ b/.github/workflows/summarize-results.yml @@ -71,10 +71,6 @@ jobs: environment: name: ado-read deployment: false - permissions: - actions: read - contents: write - id-token: write runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py index 78c0fccd8..55b034af0 100644 --- a/tests/test_review_workflows.py +++ b/tests/test_review_workflows.py @@ -281,7 +281,7 @@ def test_summarize_workflow_preserves_partial_diagnostics_and_blocks_upload() -> assert 'CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")' in workflow_text assert 'if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]' in workflow_text assert '"${CORE_SCORE_ARGS[@]}"' in workflow_text - assert workflow["jobs"]["summarize-results"]["permissions"]["actions"] == "read" + assert "permissions" not in workflow["jobs"]["summarize-results"] assert workflow_inputs["recovery-artifact-run-id"]["default"] == "" assert workflow_inputs["recovery-artifact-pattern"]["default"] == "" assert steps.index(current_download) < steps.index(recovery_download)