fix: debug benchmarks

.github/workflows/benchmark.yml

@@ -20,44 +20,48 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  run-tests:
+  run-benchmarks:
     name: Run Benchmarks
-    strategy:
-      fail-fast: false
-      matrix:
-        runs-on:
-          - ubuntu-latest
-        python-version:
-          - "3.13"
-        python-architecture:
-          - x64
-
-    runs-on: ${{ matrix.runs-on }}
+
+    runs-on: self-hosted
 
     env:
       PIP_ONLY_BINARY: numpy
 
+    # Required for miniconda to activate conda
+    defaults:
+      run:
+        shell: bash -l {0}
+
     steps:
+      - name: Clean the workspace and mamba
+        run: |
+          rm -rf * .[!.]* || echo "Nothing to clean"
+          rm -rf ~/micromamba* || echo "Nothing to clean"
+
       - uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: "Python ${{ matrix.python-version }}"
-        uses: actions/setup-python@v5
+      - name: Get micromamba
+        uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: "${{ matrix.python-version }}"
-          architecture: "${{ matrix.python-architecture }}"
-          allow-prereleases: true
+          environment-name: test-env
+          init-shell: bash
+          create-args: >-
+            python=3.13
 
       - name: Generate build files
-        run: pipx run nox -s prepare -- --headers --signatures --tests
+        run: |
+          pip install pipx
+          pipx run nox -s prepare -- --headers --signatures --tests
 
       - name: Cache awkward-cpp wheel
         id: cache-awkward-cpp-wheel
         uses: actions/cache@v4
         with:
           path: awkward-cpp/dist
-          key: ${{ github.job }}-${{ matrix.runs-on }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}-${{ hashFiles('awkward-cpp/**') }}
+          key: ${{ github.job }}-${{ hashFiles('awkward-cpp/**') }}
 
       - name: Build awkward-cpp wheel
         if: steps.cache-awkward-cpp-wheel.outputs.cache-hit != 'true'
@@ -72,6 +76,10 @@ jobs:
           files: |
             awkward-cpp/dist/*.whl
 
+      - name: Add workaround for 3.13 + cramjam
+        run: echo 'PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1' >> $GITHUB_ENV
+        shell: bash
+
       - name: Install awkward, awkward-cpp, and dependencies
         run: python -m pip install -v . ${{ steps.find-wheel.outputs.paths }} pytest-github-actions-annotate-failures
 
@@ -83,28 +91,37 @@ jobs:
 
       - name: Run Benchmark and Comparisons
         id: benchmark_and_compare
-        shell: bash
         run: |
           cd benchmarks/
           ./run_action.sh
-          echo "comparison='$(cat $BASE_OUTPUT_DIR)/comparison.md'" >> $GITHUB_OUTPUT
+          COMPARISON=$(cat $(echo $BASE_OUTPUT_DIR)/comparison.md)
+          {
+            echo "comparison<<EOF"
+            echo "${COMPARISON}"
+            echo "EOF"
+          } >> $GITHUB_OUTPUT
+          cd ..
         env:
           TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} # usually: main
           BRANCH_NAME: ${{ github.head_ref || github.ref_name }} # feature branch
-          BASE_OUTPUT_DIR: results_PR${{ inputs.pr-number }}
-          continue-on-error: true # failed benchmarking shouldn't stop the rest of the steps
+          BASE_OUTPUT_DIR: results
+        continue-on-error: true # failed benchmarking shouldn't stop the rest of the steps
 
       - name: Comment on PR
         uses: actions/github-script@v7
-        if: ${{ hashFiles(format('{0}/comparison.md', steps.benchmark_and_compare.env.BASE_OUTPUT_DIR)) != '' }} # if there's no comparison.md, we won't post anything
+        if: ${{ hashFiles('benchmarks/results/comparison.md') != '' }} # if there's no comparison.md, we won't post anything
         with:
           script: |
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,
-              body: `${{ steps.benchmark_and_compare.outputs.comparison }}`
+              body: process.env.COMPARISON
             })
+        env:
+          COMPARISON: ${{ steps.benchmark_and_compare.outputs.comparison }}
 
       - name: Cleanup Benchmark Outputs
-        run: rm -r ${{ steps.benchmark_and_compare.env.BASE_OUTPUT_DIR }}
+        run: rm -r benchmarks/$BASE_OUTPUT_DIR
+        env:
+          BASE_OUTPUT_DIR: results

benchmarks/compare.py

@@ -55,9 +55,6 @@ def compare_benchmarks(
     bm1 = {b["name"]: b for b in data1["benchmarks"]}
     bm2 = {b["name"]: b for b in data2["benchmarks"]}
 
-    file1_short = os.path.basename(file1_path)
-    file2_short = os.path.basename(file2_path)
-
     output_lines = []
 
     found_diffs = False
@@ -86,7 +83,20 @@ def compare_benchmarks(
                     "<details><summary>Show full comparison</summary>\n\n"
                 )
 
-                headers = ["Metric", f"{file1_short}", f"{file2_short}"]
+                def _parse_branch_sha(file_path):
+                    assert file_path.endswith(".json")
+                    file_path = file_path.replace(".json", "")  # remove file ending
+                    file_path = file_path.replace(
+                        os.getenv("BASE_OUTPUT_DIR", "results") + "/", "", 1
+                    )  # remove base dir
+                    branch, sha = file_path.rsplit("__", 1)  # get branch & SHA
+                    return f"branch: `{branch}` (sha: {sha})"
+
+                headers = [
+                    "Metric",
+                    _parse_branch_sha(file1_path),
+                    _parse_branch_sha(file2_path),
+                ]
                 time_unit = b1.get("time_unit", "")
                 assert time_unit == b2.get("time_unit", ""), (
                     "Can't compare difference units"

benchmarks/misc_benchmark.py

@@ -41,7 +41,6 @@ def _general_fun_benchmark(state, **kwargs):
 
 
 # extend for more misc funs
-# some of them don't make much sense to benchmark I suppose...
 FUNS = [
     ak.angle,
     ak.drop_none,

benchmarks/run_action.sh

@@ -4,10 +4,9 @@ action() {
     # This is for the HEAD@PR (including main merged)
 
     # setup output dir
-    local current_git_hash
-    current_git_hash=$(git rev-parse --verify HEAD)
-    local results_dir=${BASE_OUTPUT_DIR}/${BRANCH_NAME}__${current_git_hash}
-    local output_path_feature=${results_dir}/${bm_script}.json
+    local orig_git_hash
+    orig_git_hash=$(git rev-parse --verify HEAD)
+    local output_path_feature=${BASE_OUTPUT_DIR}/${BRANCH_NAME}__${orig_git_hash}.json
 
     # Temporarily merge the target branch
     git checkout -b pr_branch
@@ -17,11 +16,9 @@ action() {
     git merge --no-commit --no-ff origin/"${TARGET_BRANCH}" || (echo "***\nError: There are merge conflicts that need to be resolved.\n***" && false)
 
     # create
-    mkdir -p "$results_dir"
+    mkdir -p "$(dirname "${output_path_feature}")"
 
-    local bm_script="benchmark.py"
-
-    python $bm_script \
+    python benchmark.py \
         --benchmark_time_unit=ms \
         --benchmark_out="${output_path_feature}" \
         --benchmark_out_format=json
@@ -31,22 +28,23 @@ action() {
     git stash
     git checkout origin/"${TARGET_BRANCH}"
 
-    local current_git_hash
-    current_git_hash=$(git rev-parse --verify HEAD)
-    local results_dir=${BASE_OUTPUT_DIR}/${TARGET_BRANCH}__${current_git_hash}
-    local output_path_target=${results_dir}/${bm_script}.json
+    local target_git_hash
+    target_git_hash=$(git rev-parse --verify HEAD)
+    local output_path_target=${BASE_OUTPUT_DIR}/${TARGET_BRANCH}__${target_git_hash}.json
 
     # create
-    mkdir -p "$results_dir"
-
-    local bm_script="benchmark.py"
+    mkdir -p "$(dirname "${output_path_target}")"
 
-    python $bm_script \
+    python benchmark.py \
         --benchmark_time_unit=ms \
         --benchmark_out="${output_path_target}" \
         --benchmark_out_format=json
 
     # Compare both
+    # first: switch back to original commit
+    git stash
+    git checkout "${orig_git_hash}"
+
     python compare.py "${output_path_target}" "${output_path_feature}"
 }
 action "$@"