codebase-context/.github/workflows/contextbench-selected-cbm-score.yml at master · PatrickSys/codebase-context · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
name: ContextBench Selected CBM Score

on:
  push:
    branches: [master]
    paths:
      - .github/workflows/contextbench-selected-cbm-score.yml
      - scripts/contextbench-score-selected-prediction.mjs
  workflow_dispatch:

permissions:
  contents: read

jobs:
  selected-cbm-score:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      ROOT: /tmp/contextbench-selected-cbm-score
      TASK_PAYLOADS: /tmp/contextbench-selected-cbm-score/task-payloads.json
      CHECKOUT_ROOT: /tmp/contextbench-checkouts
      OFFICIAL_CONTEXTBENCH: /tmp/contextbench-selected-cbm-score/ContextBench-official
      TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v2
        with:
          version: 10
      - uses: actions/setup-node@v4
        with:
          node-version: '24'
          cache: pnpm
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install and materialize selected Go task quietly
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs"
          pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1
          python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1
          git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1
          node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1
          node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1
          node - <<'NODE'
          const fs = require('node:fs');
          const payloadPath = process.env.TASK_PAYLOADS;
          const target = process.env.TARGET_TASK_ID;
          const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8'));
          const task = payload.tasks.find((candidate) => candidate.instance_id === target);
          if (!task) throw new Error(`target task ${target} not found`);
          fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`);
          NODE
          node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1
          echo "selected_score_setup_completed"
      - name: Score selected gpt-5.4-mini-high prediction
        shell: bash
        run: node scripts/contextbench-score-selected-prediction.mjs
      - name: Upload selected score artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: contextbench-selected-cbm-score
          path: /tmp/contextbench-selected-cbm-score
          retention-days: 14