martinsrui-msft · Oct 13, 2026 · Oct 13, 2026
diff --git a/.github/workflows/bcal-evaluation.yml b/.github/workflows/bcal-evaluation.yml
@@ -1,5 +1,5 @@
 name: Evaluation with bcal
-run-name: BCal ${{ inputs.dataset || 'gold' }} (${{ inputs.external-model || 'gpt-55-chat' }}) - ${{ inputs.test-run && 'test' || 'full' }}
+run-name: BCal ${{ inputs.dataset || 'gold' }} ${{ inputs.bcal-package-selection == 'experiment-pinned' && format('experiment@{0}', inputs.bcal-version) || inputs.bcal-package-selection == 'official-pinned' && format('official@{0}', inputs.bcal-version) || 'official/latest' }} (${{ inputs.external-model || 'gpt-55-chat' }}) - ${{ inputs.test-run && 'test' || 'full' }}
 permissions:
   contents: read
   actions: write
@@ -15,16 +15,17 @@ on:
         options:
           - gold
           - challenge
-      bcal-version-mode:
-        description: "Install the latest prerelease or pin an exact BCal NuGet version"
+      bcal-package-selection:
+        description: "BCAL package selection"
         required: false
-        default: latest-prerelease
+        default: official-latest
         type: choice
         options:
-          - latest-prerelease
-          - pinned
+          - official-latest
+          - official-pinned
+          - experiment-pinned
       bcal-version:
-        description: "Exact BCal NuGet version for pinned mode; leave empty for latest-prerelease"
+        description: "Exact BCal NuGet version for pinned selections; leave empty for official-latest"
         required: false
         default: ""
         type: string
@@ -47,17 +48,133 @@ on:
 
 
 concurrency:
-  group: bcal-evaluation-${{ inputs.dataset || 'gold' }}-${{ inputs.test-run && 'test' || 'full' }}
+  group: bcal-evaluation-${{ inputs.dataset || 'gold' }}-${{ inputs.bcal-package-selection || 'official-latest' }}-${{ inputs.bcal-version || 'latest' }}-${{ inputs.test-run && 'test' || 'full' }}
   cancel-in-progress: false
 
 env:
   EVALUATION_RESULTS_DIR: evaluation_results
   NL2AL_DATASET: ${{ inputs.dataset || 'gold' }}
-  BCAL_VERSION_MODE: ${{ inputs.bcal-version-mode || 'latest-prerelease' }}
   BCAL_REQUESTED_VERSION: ${{ inputs.bcal-version }}
 
 jobs:
+  validate-bcal-package:
+    runs-on: ubuntu-latest
+    environment:
+      name: ado-read
+      deployment: false
+    permissions:
+      contents: read
+      id-token: write
+    outputs:
+      source: ${{ steps.resolve.outputs.source }}
+      branch: ${{ steps.provenance.outputs.branch || steps.resolve.outputs.branch }}
+      feed: ${{ steps.resolve.outputs.feed }}
+      version-mode: ${{ steps.resolve.outputs.version-mode }}
+    steps:
+      - name: Validate and resolve BCal package selection
+        id: resolve
+        shell: bash
+        env:
+          PACKAGE_SELECTION: ${{ inputs.bcal-package-selection || 'official-latest' }}
+          REQUESTED_VERSION: ${{ inputs.bcal-version }}
+          OFFICIAL_FEED_URL: ${{ secrets.BCAL_FEED_URL }}
+          EXPERIMENT_FEED_URL: ${{ secrets.BCAL_TEST_FEED_URL }}
+        run: |
+          set -euo pipefail
+
+          exact_version='^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$'
+
+          case "$PACKAGE_SELECTION" in
+            official-latest)
+              channel="official"
+              version_mode="latest-prerelease"
+              ;;
+            official-pinned)
+              channel="official"
+              version_mode="pinned"
+              ;;
+            experiment-pinned)
+              channel="experiment"
+              version_mode="pinned"
+              ;;
+            *)
+              echo "::error::Unknown bcal-package-selection: $PACKAGE_SELECTION"
+              exit 1
+              ;;
+          esac
+
+          case "$version_mode" in
+            latest-prerelease)
+              if [[ -n "$REQUESTED_VERSION" ]]; then
+                echo "::error::bcal-version must be empty for official-latest."
+                exit 1
+              fi
+              ;;
+            pinned)
+              if [[ ! "$REQUESTED_VERSION" =~ $exact_version ]]; then
+                echo "::error::Pinned package selections require bcal-version to be an exact NuGet version."
+                exit 1
+              fi
+              ;;
+          esac
+
+          if [[ "$OFFICIAL_FEED_URL" != https://* ]]; then
+            echo "::error::BCAL_FEED_URL must be a non-empty HTTPS Azure Artifacts URL."
+            exit 1
+          fi
+
+          case "$channel" in
+            official)
+              branch="master"
+              feed="DynamicsSMBPackages"
+              ;;
+            experiment)
+              if [[ "$EXPERIMENT_FEED_URL" != https://* ]]; then
+                echo "::error::BCAL_TEST_FEED_URL must be a non-empty HTTPS Azure Artifacts URL."
+                exit 1
+              fi
+              if [[ "$OFFICIAL_FEED_URL" == "$EXPERIMENT_FEED_URL" ]]; then
+                echo "::error::Official and experimental BCAL feed URLs must differ."
+                exit 1
+              fi
+              branch=""
+              feed="DynamicsSMBTest@Local"
+              ;;
+          esac
+
+          echo "source=$channel" >> "$GITHUB_OUTPUT"
+          echo "branch=$branch" >> "$GITHUB_OUTPUT"
+          echo "feed=$feed" >> "$GITHUB_OUTPUT"
+          echo "version-mode=$version_mode" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout repository for experimental package provenance
+        if: ${{ steps.resolve.outputs.source == 'experiment' }}
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Azure Login (OIDC) for experimental package provenance
+        if: ${{ steps.resolve.outputs.source == 'experiment' }}
+        uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          allow-no-subscriptions: true
+
+      - name: Verify experimental package provenance
+        if: ${{ steps.resolve.outputs.source == 'experiment' }}
+        id: provenance
+        shell: pwsh
+        env:
+          BCAL_PACKAGE_ID: ${{ secrets.BCAL_PACKAGE_ID }}
+          BCAL_REQUESTED_VERSION: ${{ inputs.bcal-version }}
+          BCAL_TEST_FEED_URL: ${{ secrets.BCAL_TEST_FEED_URL }}
+        run: |
+          .\scripts\Resolve-BCalPackageProvenance.ps1 `
+            -PackageId "$env:BCAL_PACKAGE_ID" `
+            -Version "$env:BCAL_REQUESTED_VERSION" `
+            -TestFeedUrl "$env:BCAL_TEST_FEED_URL"
+
   get-entries:
+    needs: validate-bcal-package
     uses: $/.github/workflows/get-entries.yml
     with:
       test-run: ${{ inputs.test-run || false }}
@@ -66,7 +183,7 @@ jobs:
 
   evaluate-with-bcal:
     runs-on: windows-latest
-    needs: get-entries
+    needs: [validate-bcal-package, get-entries]
     outputs:
       results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
     env:
@@ -98,8 +215,8 @@ jobs:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
           tenant-id: ${{ secrets.AZURE_TENANT_ID }}
           package-id: ${{ secrets.BCAL_PACKAGE_ID }}
-          feed-url: ${{ secrets.BCAL_FEED_URL }}
-          version-mode: ${{ env.BCAL_VERSION_MODE }}
+          feed-url: ${{ needs.validate-bcal-package.outputs.source == 'experiment' && secrets.BCAL_TEST_FEED_URL || secrets.BCAL_FEED_URL }}
+          version-mode: ${{ needs.validate-bcal-package.outputs.version-mode }}
           version: ${{ env.BCAL_REQUESTED_VERSION }}
 
       - name: Build bc-eval LLM API bridge venv
@@ -174,6 +291,8 @@ jobs:
         env:
           BCAL_LLM_MODEL: ${{ inputs.external-model || 'gpt-55-chat' }}
           BCAL_PACKAGE_VERSION: ${{ steps.install-bcal.outputs.version }}
+          BCAL_PACKAGE_CHANNEL: ${{ needs.validate-bcal-package.outputs.source }}
+          BCAL_PACKAGE_BRANCH: ${{ needs.validate-bcal-package.outputs.branch }}
           LLM_API_TENANT_ID: ${{ secrets.CAPI_TENANT_ID }}
           LLM_API_CLIENT_ID: ${{ secrets.CAPI_CLIENT_ID }}
         shell: pwsh
@@ -198,8 +317,8 @@ jobs:
           retention-days: ${{ inputs.test-run && 1 || 30 }}
 
   summarize-results:
-    needs: [get-entries, evaluate-with-bcal]
-    if: ${{ always() && !cancelled() && needs.get-entries.result == 'success' && needs.get-entries.outputs.entries != '[]' }}
+    needs: [validate-bcal-package, get-entries, evaluate-with-bcal]
+    if: ${{ always() && !cancelled() && needs.validate-bcal-package.result == 'success' && needs.get-entries.result == 'success' && needs.get-entries.outputs.entries != '[]' }}
     uses: $/.github/workflows/summarize-results.yml
     permissions:
       contents: write
@@ -211,6 +330,9 @@ jobs:
       mock: ${{ inputs.test-run || false }}
       category: nl2al
       dataset: ${{ inputs.dataset || 'gold' }}
+      bcal-package-channel: ${{ needs.validate-bcal-package.outputs.source }}
+      bcal-package-branch: ${{ needs.validate-bcal-package.outputs.branch }}
+      bcal-package-feed: ${{ needs.validate-bcal-package.outputs.feed }}
       skip-leaderboard: true
       expected-entries: ${{ needs.get-entries.outputs.entries }}
     secrets: inherit
diff --git a/.github/workflows/summarize-results.yml b/.github/workflows/summarize-results.yml
@@ -29,6 +29,21 @@ on:
         required: false
         type: string
         default: ""
+      bcal-package-channel:
+        description: "Optional BCAL package channel label"
+        required: false
+        type: string
+        default: ""
+      bcal-package-branch:
+        description: "Optional BCAL source branch label"
+        required: false
+        type: string
+        default: ""
+      bcal-package-feed:
+        description: "Optional BCAL package feed label"
+        required: false
+        type: string
+        default: ""
       expected-entries:
         description: "Optional JSON array of selected entry IDs; require complete results before scoring upload"
         required: false
@@ -67,6 +82,9 @@ jobs:
       EVALUATION_MODEL: ${{ inputs.model }}
       EVALUATION_GIT_REF: ${{ inputs.git-ref || github.ref_name }}
       EXPECTED_ENTRIES: ${{ inputs.expected-entries }}
+      BCAL_PACKAGE_CHANNEL: ${{ inputs.bcal-package-channel }}
+      BCAL_PACKAGE_BRANCH: ${{ inputs.bcal-package-branch }}
+      BCAL_PACKAGE_FEED: ${{ inputs.bcal-package-feed }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -148,6 +166,17 @@ jobs:
           BRANCH_TAG="${BRANCH_TAG//./-}"
           RUN_NAME="${EVALUATION_AGENT} (${EVALUATION_MODEL}) - #${GITHUB_RUN_ID}"
           RUN_TAGS="${EVALUATION_AGENT},${MODEL_TAG},${BRANCH_TAG}"
+          if [[ -n "$BCAL_PACKAGE_CHANNEL" ]]; then
+            CHANNEL_TAG="$(printf '%s' "$BCAL_PACKAGE_CHANNEL" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')"
+            PACKAGE_BRANCH_TAG="$(printf '%s' "$BCAL_PACKAGE_BRANCH" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')"
+            FEED_TAG="$(printf '%s' "$BCAL_PACKAGE_FEED" | tr -c '[:alnum:]_-' '-' | sed -E 's/^-+|-+$//g')"
+            if [[ -z "$CHANNEL_TAG" || -z "$PACKAGE_BRANCH_TAG" || -z "$FEED_TAG" ]]; then
+              echo "::error::BCAL package provenance cannot be converted to non-empty run tags."
+              exit 1
+            fi
+            RUN_NAME="${RUN_NAME} [${BCAL_PACKAGE_CHANNEL}:${BCAL_PACKAGE_BRANCH}]"
+            RUN_TAGS="${RUN_TAGS},bcal-channel-${CHANNEL_TAG},bcal-branch-${PACKAGE_BRANCH_TAG},bcal-feed-${FEED_TAG}"
+          fi
           if [[ -n "$NL2AL_DATASET" ]]; then
             RUN_NAME="${RUN_NAME} [${NL2AL_DATASET}]"
             RUN_TAGS="${RUN_TAGS},dataset-${NL2AL_DATASET}"

diff --git a/docs/bcal-datasets.md b/docs/bcal-datasets.md
@@ -11,9 +11,19 @@ The **Evaluation with bcal** workflow defaults to `gold` for both scheduled and
 
 ## BCAL package version
 
-The workflow defaults to `latest-prerelease`, preserving the existing installation behavior. Manual runs can instead select `pinned` and provide an exact NuGet version in `bcal-version`, including an optional fourth numeric component, prerelease suffix, or build metadata. Empty, floating, and range values are rejected before `dotnet` runs.
+The workflow combines channel and version mode into one package selection:
 
-Every matrix job resolves and installs its own package version. The installer reads the structured global-tool list after installation and passes the verified actual NuGet version to evaluation. Raw results record that observed value as `agent_version`, including agent failures and timeouts; they never record `latest`, `unknown`, or an unchecked requested string. Because a feed can publish between matrix jobs, use a pinned version when every case must run with the same package.
+- **`official-latest`** (default): installs the latest prerelease from the official feed and records the BCAL source branch as `master`.
+- **`official-pinned`**: installs an exact version from the official feed.
+- **`experiment-pinned`**: installs an exact package from the configured experimental `@Local` feed and records the originating BC-DeveloperExperience branch or PR.
+
+Pinned selections require an exact NuGet version in `bcal-version`, including an optional fourth numeric component, prerelease suffix, or build metadata. `official-latest` requires the version field to be empty. Empty pinned versions, floating versions, and ranges are rejected before `dotnet` runs.
+
+There is no experimental-latest option because the feed contains packages from multiple branches and PRs. `experiment-pinned` requires the `BCAL_TEST_FEED_URL` repository secret to point to the experimental `@Local` feed.
+
+Before starting the matrix, BC-Bench verifies that the exact experimental package was published directly from BC-DeveloperExperience and derives its branch or `pr/<number>` identity.
+
+Every official matrix job resolves and installs its own package version; experimental jobs install the exact requested pin. Raw results record the verified installed version as `agent_version`.
 
 Local `evaluate bcal` performs the same global-tool discovery. Use `--bcal-version <exact-version>` or set `BCAL_PACKAGE_VERSION` to require the installed version to match an expected value:
 
@@ -57,6 +67,10 @@ Each named-panel bc-eval row includes:
 
 | Metadata | Value |
 |---|---|
+| `agent_version` | Actual installed BCAL NuGet package version |
+| `bcal_package_source` | `official` or `experiment` |
+| `bcal_package_branch` | `master` or the provenance-derived BC-DeveloperExperience source branch |
+| `bcal_package_feed` | `DynamicsSMBPackages` or `DynamicsSMBTest@Local` |
 | `dataset`, `dataset_version`, `dataset_sha256` | Exact panel identity |
 | `dataset_mode` | `single_turn` |
 | `dataset_turn_count` | `1` |
@@ -65,6 +79,8 @@ Each named-panel bc-eval row includes:
 
 The row tags include the dataset, reviewed version, single-turn mode, evaluation scope, and available area/family/tier classifications.
 
+Do not pool different dataset hashes or BCAL package variants as if they were the same population. Rows without package provenance are older/unclassified results, not automatically official or master.
+
 ## Complete-panel safety
 
 The BCAL workflow passes the exact selected matrix entry IDs to result summarization. Before any bc-eval export, summarization compares those IDs with the result artifacts and rejects:

diff --git a/scripts/Install-BCalTool.ps1 b/scripts/Install-BCalTool.ps1
@@ -31,7 +31,7 @@ if ($VersionMode -eq "pinned") {
 else {
     $installArguments += "--prerelease"
 }
-$installArguments += @("--add-source", $FeedUrl)
+$installArguments += @("--source", $FeedUrl)
 
 for ($attempt = 1; $attempt -le 3; $attempt++) {
     & dotnet @installArguments