martinsrui-msft · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 8, 2026
diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml
@@ -21,6 +21,21 @@ on:
         required: false
         default: true
         type: boolean
+      recovery-source-run-id:
+        description: "Prior workflow run ID whose artifacts should be combined with one recovered entry"
+        required: false
+        default: ""
+        type: string
+      recovery-source-run-attempt:
+        description: "Prior workflow run attempt containing the reusable artifacts"
+        required: false
+        default: ""
+        type: string
+      recovery-entry:
+        description: "Single NL2AL entry to rerun while reusing artifacts from the prior attempt"
+        required: false
+        default: ""
+        type: string
   schedule:
     - cron: "0 3 * * 0"
 
@@ -31,20 +46,99 @@ concurrency:
 
 env:
   EVALUATION_RESULTS_DIR: evaluation_results
+  BCCONTAINERHELPER_VERSION: 6.1.18
 
 jobs:
   get-entries:
     uses: $/.github/workflows/get-entries.yml
     with:
-      test-run: ${{ inputs.test-run || false }}
+      test-run: ${{ inputs.recovery-entry == '' && inputs.test-run }}
       category: nl2al
 
+  select-entries:
+    runs-on: ubuntu-latest
+    needs: get-entries
+    outputs:
+      entries: ${{ steps.select.outputs.entries }}
+    steps:
+      - name: Select entries for evaluation
+        id: select
+        env:
+          ALL_ENTRIES: ${{ needs.get-entries.outputs.entries }}
+          RECOVERY_SOURCE_RUN_ID: ${{ inputs.recovery-source-run-id }}
+          RECOVERY_SOURCE_RUN_ATTEMPT: ${{ inputs.recovery-source-run-attempt }}
+          RECOVERY_ENTRY: ${{ inputs.recovery-entry }}
+          TEST_RUN: ${{ inputs.test-run }}
+        run: |
+          if [[ -z "$RECOVERY_SOURCE_RUN_ID" && -z "$RECOVERY_SOURCE_RUN_ATTEMPT" && -z "$RECOVERY_ENTRY" ]]; then
+            echo "entries=$ALL_ENTRIES" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [[ -z "$RECOVERY_SOURCE_RUN_ID" || -z "$RECOVERY_SOURCE_RUN_ATTEMPT" || -z "$RECOVERY_ENTRY" ]]; then
+            echo "::error::recovery-source-run-id, recovery-source-run-attempt, and recovery-entry must be provided together."
+            exit 1
+          fi
+          if [[ "$TEST_RUN" == "true" ]]; then
+            echo "::error::Recovery runs must set test-run to false so the complete result is scored and uploaded."
+            exit 1
+          fi
+          if ! [[ "$RECOVERY_SOURCE_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then
+            echo "::error::recovery-source-run-id must be a positive integer."
+            exit 1
+          fi
+          if ! [[ "$RECOVERY_SOURCE_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]; then
+            echo "::error::recovery-source-run-attempt must be a positive integer."
+            exit 1
+          fi
+          if ! jq -e --arg entry "$RECOVERY_ENTRY" 'index($entry) != null' <<< "$ALL_ENTRIES" > /dev/null; then
+            echo "::error::Recovery entry '$RECOVERY_ENTRY' is not in the full NL2AL dataset."
+            exit 1
+          fi
+
+          echo "entries=$(jq -cn --arg entry "$RECOVERY_ENTRY" '[$entry]')" >> "$GITHUB_OUTPUT"
+
+  prepare-bccontainerhelper:
+    runs-on: windows-latest
+    needs: select-entries
+    if: needs.select-entries.outputs.entries != '[]'
+    steps:
+      - name: Cache pinned BcContainerHelper
+        id: cache-bccontainerhelper
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~\Documents\PowerShell\Modules\BcContainerHelper
+          key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }}
+
+      - name: Download pinned BcContainerHelper
+        if: steps.cache-bccontainerhelper.outputs.cache-hit != 'true'
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $moduleRoot = Join-Path $HOME 'Documents\PowerShell\Modules'
+          New-Item -ItemType Directory -Path $moduleRoot -Force | Out-Null
+          Save-Module `
+            -Name BcContainerHelper `
+            -RequiredVersion $env:BCCONTAINERHELPER_VERSION `
+            -Repository PSGallery `
+            -Path $moduleRoot `
+            -Force `
+            -ErrorAction Stop
+
+      - name: Verify pinned BcContainerHelper
+        shell: pwsh
+        run: |
+          Import-Module BcContainerHelper -RequiredVersion $env:BCCONTAINERHELPER_VERSION -Force -DisableNameChecking
+          if ((Get-Module BcContainerHelper).Version -ne [Version]$env:BCCONTAINERHELPER_VERSION) {
+            throw "Expected BcContainerHelper $env:BCCONTAINERHELPER_VERSION."
+          }
+
   evaluate-with-bcal:
     runs-on: windows-latest
-    needs: get-entries
+    needs: [select-entries, prepare-bccontainerhelper]
     outputs:
       results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
-    if: needs.get-entries.outputs.entries != '[]'
+    if: needs.select-entries.outputs.entries != '[]'
     environment:
       name: ado-read
       deployment: false
@@ -56,7 +150,7 @@ jobs:
       fail-fast: false
       max-parallel: 32
       matrix:
-        entry: ${{ fromJson(needs.get-entries.outputs.entries) }}
+        entry: ${{ fromJson(needs.select-entries.outputs.entries) }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -136,6 +230,13 @@ jobs:
         id: bcversion
         run: uv run bcbench dataset version "${{ matrix.entry }}" --category nl2al --github-output version
 
+      - name: Restore pinned BcContainerHelper
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~\Documents\PowerShell\Modules\BcContainerHelper
+          key: bcal-bccontainerhelper-${{ runner.os }}-${{ env.BCCONTAINERHELPER_VERSION }}
+          fail-on-cache-miss: true
+
       # Cache BcContainerHelper's artifact folder (its hardcoded default path) keyed by this entry's BC version
       - name: Cache BC sandbox artifacts
         id: bccache
@@ -148,7 +249,6 @@ jobs:
         if: steps.bccache.outputs.cache-hit != 'true'
         shell: pwsh
         run: |
-          Install-Module -Name BcContainerHelper -Force -AllowClobber -AllowPrerelease
           .\scripts\Download-BCSymbols.ps1 -Category nl2al -InstanceId "${{ matrix.entry }}"
 
       - name: Azure Login (OIDC) for CAPI Key Vault
@@ -186,6 +286,7 @@ jobs:
           CAPI_CLIENT_ID: ${{ secrets.CAPI_CLIENT_ID }}
           CAPI_CERTIFICATE_KEY_VAULT_NAME: ${{ secrets.CAPI_CERTIFICATE_KEY_VAULT_NAME }}
           CAPI_CERTIFICATE_NAME: ${{ secrets.CAPI_CERTIFICATE_NAME }}
+          PYTHONPATH: ${{ github.workspace }}\src
         shell: pwsh
         run: |
           $env:CAPI_CERT_FILE = "$env:RUNNER_TEMP\capi-cert.pfx"
@@ -200,14 +301,16 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:
-          name: evaluation-results-${{ github.run_id }}-${{ matrix.entry }}
+          name: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.entry }}
           path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
           retention-days: ${{ inputs.test-run && 1 || 30 }}
 
   summarize-results:
-    needs: evaluate-with-bcal
+    needs: [get-entries, select-entries, evaluate-with-bcal]
+    if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.select-entries.result == 'success' && needs.get-entries.outputs.entry-count != '0' }}
     uses: $/.github/workflows/summarize-results.yml
     permissions:
+      actions: read
       contents: write
       id-token: write
     with:
@@ -217,4 +320,9 @@ jobs:
       mock: ${{ inputs.test-run || false }}
       category: nl2al
       skip-leaderboard: true
+      expected-total: ${{ fromJSON(needs.get-entries.outputs.entry-count) }}
+      artifact-pattern: evaluation-results-${{ github.run_id }}-${{ github.run_attempt }}-*
+      recovery-artifact-run-id: ${{ inputs.recovery-source-run-id }}
+      recovery-artifact-pattern: ${{ inputs.recovery-source-run-id != '' && format('evaluation-results-{0}-{1}-*', inputs.recovery-source-run-id, inputs.recovery-source-run-attempt) || '' }}
+      allow-unscored-results: true
     secrets: inherit
diff --git a/.github/workflows/get-entries.yml b/.github/workflows/get-entries.yml
@@ -24,6 +24,9 @@ on:
       entries:
         description: JSON array of dataset entries
         value: ${{ jobs.get-entries.outputs.entries }}
+      entry-count:
+        description: Number of dataset entries
+        value: ${{ jobs.get-entries.outputs.entry-count }}
       runner:
         description: GitHub Actions runner label to evaluate this category on
         value: ${{ jobs.get-entries.outputs.runner }}
@@ -39,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       entries: ${{ steps.get-entries.outputs.entries }}
+      entry-count: ${{ steps.get-entries.outputs.entry-count }}
       runner: ${{ steps.runtime-config.outputs.runner }}
       requires-container: ${{ steps.runtime-config.outputs.requires-container }}
       requires-repo: ${{ steps.runtime-config.outputs.requires-repo }}
@@ -58,7 +62,7 @@ jobs:
       - name: Get entries for matrix
         id: get-entries
         run: |
-          cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries"
+          cmd="uv run bcbench dataset list --category ${{ inputs.category }} --github-output entries --count-output entry-count"
 
           if [[ "${{ inputs.modified-only }}" == "true" ]]; then
             cmd="$cmd --modified-only"

diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml
@@ -34,20 +34,43 @@ on:
         required: false
         type: string
         default: ""
+      expected-total:
+        description: "Expected unique matrix entry count; negative disables completeness enforcement"
+        required: false
+        type: number
+        default: -1
+      artifact-pattern:
+        description: "Artifact glob to download"
+        required: false
+        type: string
+        default: "*"
+      recovery-artifact-run-id:
+        description: "Prior workflow run ID containing artifacts to merge into this run"
+        required: false
+        type: string
+        default: ""
+      recovery-artifact-pattern:
+        description: "Artifact glob to download from the recovery source run"
+        required: false
+        type: string
+        default: ""
+      allow-unscored-results:
+        description: "Allow complete result sets containing explicit unscored infrastructure errors"
+        required: false
+        type: boolean
+        default: false
 
 env:
   BCEVAL_RESULT_FILE: bceval_results.jsonl
   SUMMARY_OUTPUT_FILE: evaluation_summary.json
+  COMPLETENESS_OUTPUT_FILE: evaluation_completeness.json
 
 jobs:
   summarize-results:
     name: Results
     environment:
       name: ado-read
       deployment: false
-    permissions:
-      contents: write
-      id-token: write
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -58,33 +81,51 @@ jobs:
 
       - name: Download all evaluation results
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        continue-on-error: ${{ inputs.expected-total >= 0 }}
         with:
           path: ${{ inputs.results-dir }}/
+          pattern: ${{ inputs.artifact-pattern }}
           merge-multiple: true
 
-      - name: Summarize evaluation results
-        run: uv run bcbench result summarize --category "${{ inputs.category }}" --result-dir "${{ inputs.results-dir }}" --bceval-output "${{ env.BCEVAL_RESULT_FILE }}" --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}" --git-ref "${{ inputs.git-ref || github.ref_name }}"
-
-      - name: Upload evaluation summary to artifacts
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      - name: Download recovery evaluation results
+        if: inputs.recovery-artifact-run-id != ''
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: evaluation-summary
-          path: ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}
-          if-no-files-found: error
-          retention-days: ${{ inputs.mock && 1 || 30 }}
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ inputs.recovery-artifact-run-id }}
+          path: ${{ inputs.results-dir }}/${{ github.run_id }}/recovered/
+          pattern: ${{ inputs.recovery-artifact-pattern }}
+          merge-multiple: true
+
+      - name: Summarize evaluation results
+        id: summarize
+        run: >-
+          uv run bcbench result summarize
+          --category "${{ inputs.category }}"
+          --result-dir "${{ inputs.results-dir }}"
+          --bceval-output "${{ env.BCEVAL_RESULT_FILE }}"
+          --summary-output "${{ env.SUMMARY_OUTPUT_FILE }}"
+          --completeness-output "${{ env.COMPLETENESS_OUTPUT_FILE }}"
+          --expected-total "${{ inputs.expected-total }}"
+          --git-ref "${{ inputs.git-ref || github.ref_name }}"
+          --github-output
 
       - name: Azure Login with OIDC for bceval package
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           allow-no-subscriptions: true
 
       - name: Resolve bceval evaluator config for category
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         id: bceval
         run: uv run bcbench category bceval-config --category "${{ inputs.category }}"
 
-      - name: Upload result using bceval
+      - name: Score and upload complete result using bceval
+        if: steps.summarize.outputs.has-results == 'true' && steps.summarize.outputs.complete == 'true'
         env:
           BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }}
           BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
@@ -96,11 +137,14 @@ jobs:
 
           uv tool install bc-eval[capi]==0.3.14 --python 3.12 --index "https://anything:${ADO_TOKEN}@dynamicssmb2.pkgs.visualstudio.com/1fcb79e7-ab07-432a-a3c6-6cf5a88ba4a5/_packaging/BC_PythonFeed/pypi/simple/"
 
-          # Upload summary using bc-eval
           MODEL_TAG="${{ inputs.model }}"
           MODEL_TAG="${MODEL_TAG//./-}"
           BRANCH_TAG="${{ inputs.git-ref || github.ref_name }}"
           BRANCH_TAG="${BRANCH_TAG//./-}"
+          CORE_SCORE_ARGS=(--core-score "${{ steps.bceval.outputs.core_score }}")
+          if [[ "${{ inputs.allow-unscored-results }}" == "true" ]]; then
+            CORE_SCORE_ARGS=()
+          fi
           bceval metrics calculate \
             --feature-name "BC-Bench" \
             --eval-suite-name "${{ inputs.category }}" \
@@ -110,13 +154,25 @@ jobs:
             --input-file "${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}" \
             --evaluator-definitions "${{ github.workspace }}/evaluator/scores.py" \
             --evaluators "${{ steps.bceval.outputs.evaluators }}" \
-            --core-score "${{ steps.bceval.outputs.core_score }}" \
+            "${CORE_SCORE_ARGS[@]}" \
             --metric-definitions "${{ github.workspace }}/evaluator/metrics.py" \
             --metrics "bc_bench_metrics" \
             --use-capi ${{ !inputs.mock && '--storage braintrust --storage kusto' || '' }}
 
+      - name: Upload evaluation summary to artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: evaluation-summary-${{ github.run_attempt }}
+          path: |
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.SUMMARY_OUTPUT_FILE }}
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}
+            ${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.BCEVAL_RESULT_FILE }}
+          if-no-files-found: error
+          retention-days: ${{ inputs.mock && 1 || 30 }}
+
       - name: Update leaderboard in a new branch
-        if: ${{ !inputs.mock && !inputs.skip-leaderboard }}
+        if: ${{ !inputs.mock && !inputs.skip-leaderboard && steps.summarize.outputs.complete == 'true' }}
         run: |
           git fetch origin main
 
@@ -150,3 +206,13 @@ jobs:
           git commit -m "Update leaderboard: ${{ inputs.agent }} (${{ inputs.model }}) - Run ${{ github.run_id }}"
 
           git push --set-upstream origin $branchName
+
+      - name: Require complete matrix
+        if: ${{ always() && inputs.expected-total >= 0 }}
+        run: |
+          completeness_file="${{ inputs.results-dir }}/${{ github.run_id }}/${{ env.COMPLETENESS_OUTPUT_FILE }}"
+          if [[ ! -f "$completeness_file" ]]; then
+            echo "::error::Completeness metadata was not produced."
+            exit 1
+          fi
+          uv run bcbench result require-complete "$completeness_file"