BC-Bench/.github/workflows/CI.yml at main · microsoft/BC-Bench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

permissions:
  contents: read

env:
  EVALUATION_RESULTS_DIR: evaluation_results

jobs:
  lint-and-test:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Python with UV
        uses: $/.github/actions/setup-python-uv
        with:
          full-install: true

      - name: Run pre-commit
        run: uv run pre-commit run --all-files --show-diff-on-failure --color=always
        env:
          # Render ruff lint findings as inline PR annotations (CI-only; local runs stay human-readable)
          RUFF_OUTPUT_FORMAT: github

      - name: Run tests
        run: uv run pytest

  coverage:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      code-quality: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # Test the PR head rather than the merge commit so coverage lines map onto the PR diff; lint-and-test covers the merge
          # https://docs.github.com/en/code-security/how-tos/maintain-quality-code/set-up-code-coverage#manual-setup
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Setup Python with UV
        uses: $/.github/actions/setup-python-uv
        with:
          full-install: true

      - name: Run tests with coverage
        run: uv run pytest --cov --cov-report=term-missing --cov-report=xml

      - name: Upload coverage to GitHub Code Quality
        # The action skips fork PRs and branches without a PR itself; dispatch runs have no PR number, which the API rejects
        if: github.event_name != 'workflow_dispatch'
        uses: actions/upload-code-coverage@2b21a77928be8d5168c2b9581a67f2adbebacc52 # v1.4.4
        with:
          file: coverage.xml
          language: Python
          label: code-coverage/pytest

  e2e:
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Python with UV
        uses: $/.github/actions/setup-python-uv

      - name: Checkout BCQuality
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: microsoft/BCQuality
          path: bcquality

      - name: Run E2E tests
        run: uv run pytest -m e2e
        env:
          BCQUALITY_ROOT: ${{ github.workspace }}/bcquality

  select-category:
    runs-on: ubuntu-latest
    outputs:
      category: ${{ steps.random.outputs.category }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Python with UV
        uses: $/.github/actions/setup-python-uv

      - name: Select random category
        id: random
        shell: pwsh
        run: |
          $categories = (uv run bcbench category list) -split "`n" | Where-Object { $_ }
          $selected = $categories | Get-Random
          echo "category=$selected" >> $env:GITHUB_OUTPUT

  get-entries:
    needs: select-category
    uses: $/.github/workflows/get-entries.yml
    with:
      test-run: true
      category: ${{ needs.select-category.outputs.category }}

  mock-evaluation:
    runs-on: ubuntu-latest
    needs: [get-entries, select-category]
    if: needs.get-entries.outputs.entries != '[]'
    outputs:
      results-dir: ${{ env.EVALUATION_RESULTS_DIR }}
    strategy:
      fail-fast: false
      matrix:
        entry: ${{ fromJson(needs.get-entries.outputs.entries) }}
    name: Test Run for ${{ matrix.entry }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Setup Python with UV
        uses: $/.github/actions/setup-python-uv

      - name: Run mock evaluation for ${{ matrix.entry }}
        run: uv run bcbench evaluate mock "${{ matrix.entry }}" --category ${{ needs.select-category.outputs.category }} --output-dir evaluation_results --run-id ${{ github.run_id }}

      - name: Upload mock evaluation results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ matrix.entry }}
          path: ${{ env.EVALUATION_RESULTS_DIR }}/**/*.jsonl
          retention-days: 1

  summarize-results:
    needs: [mock-evaluation, select-category]
    uses: $/.github/workflows/summarize-results.yml
    permissions:
      contents: write
      id-token: write
    with:
      results-dir: ${{ needs.mock-evaluation.outputs.results-dir }}
      model: ${{ github.run_id }}
      agent: "mock-agent"
      mock: true
      category: ${{ needs.select-category.outputs.category }}
    secrets: inherit