diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 1f8daaddc2..0051faffc6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,44 +20,48 @@ concurrency: cancel-in-progress: true jobs: - run-tests: + run-benchmarks: name: Run Benchmarks - strategy: - fail-fast: false - matrix: - runs-on: - - ubuntu-latest - python-version: - - "3.13" - python-architecture: - - x64 - - runs-on: ${{ matrix.runs-on }} + + runs-on: self-hosted env: PIP_ONLY_BINARY: numpy + # Required for miniconda to activate conda + defaults: + run: + shell: bash -l {0} + steps: + - name: Clean the workspace and mamba + run: | + rm -rf * .[!.]* || echo "Nothing to clean" + rm -rf ~/micromamba* || echo "Nothing to clean" + - uses: actions/checkout@v4 with: submodules: true - - name: "Python ${{ matrix.python-version }}" - uses: actions/setup-python@v5 + - name: Get micromamba + uses: mamba-org/setup-micromamba@v2 with: - python-version: "${{ matrix.python-version }}" - architecture: "${{ matrix.python-architecture }}" - allow-prereleases: true + environment-name: test-env + init-shell: bash + create-args: >- + python=3.13 - name: Generate build files - run: pipx run nox -s prepare -- --headers --signatures --tests + run: | + pip install pipx + pipx run nox -s prepare -- --headers --signatures --tests - name: Cache awkward-cpp wheel id: cache-awkward-cpp-wheel uses: actions/cache@v4 with: path: awkward-cpp/dist - key: ${{ github.job }}-${{ matrix.runs-on }}-${{ matrix.python-version }}-${{ matrix.python-architecture }}-${{ hashFiles('awkward-cpp/**') }} + key: ${{ github.job }}-${{ hashFiles('awkward-cpp/**') }} - name: Build awkward-cpp wheel if: steps.cache-awkward-cpp-wheel.outputs.cache-hit != 'true' @@ -72,6 +76,10 @@ jobs: files: | awkward-cpp/dist/*.whl + - name: Add workaround for 3.13 + cramjam + run: echo 'PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1' >> $GITHUB_ENV + shell: bash + - name: Install awkward, awkward-cpp, and dependencies run: python -m pip install -v . ${{ steps.find-wheel.outputs.paths }} pytest-github-actions-annotate-failures @@ -83,28 +91,37 @@ jobs: - name: Run Benchmark and Comparisons id: benchmark_and_compare - shell: bash run: | cd benchmarks/ ./run_action.sh - echo "comparison='$(cat $BASE_OUTPUT_DIR)/comparison.md'" >> $GITHUB_OUTPUT + COMPARISON=$(cat $(echo $BASE_OUTPUT_DIR)/comparison.md) + { + echo "comparison<> $GITHUB_OUTPUT + cd .. env: TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} # usually: main BRANCH_NAME: ${{ github.head_ref || github.ref_name }} # feature branch - BASE_OUTPUT_DIR: results_PR${{ inputs.pr-number }} - continue-on-error: true # failed benchmarking shouldn't stop the rest of the steps + BASE_OUTPUT_DIR: results + continue-on-error: true # failed benchmarking shouldn't stop the rest of the steps - name: Comment on PR uses: actions/github-script@v7 - if: ${{ hashFiles(format('{0}/comparison.md', steps.benchmark_and_compare.env.BASE_OUTPUT_DIR)) != '' }} # if there's no comparison.md, we won't post anything + if: ${{ hashFiles('benchmarks/results/comparison.md') != '' }} # if there's no comparison.md, we won't post anything with: script: | github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: `${{ steps.benchmark_and_compare.outputs.comparison }}` + body: process.env.COMPARISON }) + env: + COMPARISON: ${{ steps.benchmark_and_compare.outputs.comparison }} - name: Cleanup Benchmark Outputs - run: rm -r ${{ steps.benchmark_and_compare.env.BASE_OUTPUT_DIR }} + run: rm -r benchmarks/$BASE_OUTPUT_DIR + env: + BASE_OUTPUT_DIR: results diff --git a/benchmarks/compare.py b/benchmarks/compare.py index 0998bf042a..66c29240ab 100644 --- a/benchmarks/compare.py +++ b/benchmarks/compare.py @@ -55,9 +55,6 @@ def compare_benchmarks( bm1 = {b["name"]: b for b in data1["benchmarks"]} bm2 = {b["name"]: b for b in data2["benchmarks"]} - file1_short = os.path.basename(file1_path) - file2_short = os.path.basename(file2_path) - output_lines = [] found_diffs = False @@ -86,7 +83,20 @@ def compare_benchmarks( "
Show full comparison\n\n" ) - headers = ["Metric", f"{file1_short}", f"{file2_short}"] + def _parse_branch_sha(file_path): + assert file_path.endswith(".json") + file_path = file_path.replace(".json", "") # remove file ending + file_path = file_path.replace( + os.getenv("BASE_OUTPUT_DIR", "results") + "/", "", 1 + ) # remove base dir + branch, sha = file_path.rsplit("__", 1) # get branch & SHA + return f"branch: `{branch}` (sha: {sha})" + + headers = [ + "Metric", + _parse_branch_sha(file1_path), + _parse_branch_sha(file2_path), + ] time_unit = b1.get("time_unit", "") assert time_unit == b2.get("time_unit", ""), ( "Can't compare difference units" diff --git a/benchmarks/misc_benchmark.py b/benchmarks/misc_benchmark.py index 07c75d54c4..0419ec9126 100644 --- a/benchmarks/misc_benchmark.py +++ b/benchmarks/misc_benchmark.py @@ -41,7 +41,6 @@ def _general_fun_benchmark(state, **kwargs): # extend for more misc funs -# some of them don't make much sense to benchmark I suppose... FUNS = [ ak.angle, ak.drop_none, diff --git a/benchmarks/run_action.sh b/benchmarks/run_action.sh index 807526b46a..7131f2e8ed 100755 --- a/benchmarks/run_action.sh +++ b/benchmarks/run_action.sh @@ -4,10 +4,9 @@ action() { # This is for the HEAD@PR (including main merged) # setup output dir - local current_git_hash - current_git_hash=$(git rev-parse --verify HEAD) - local results_dir=${BASE_OUTPUT_DIR}/${BRANCH_NAME}__${current_git_hash} - local output_path_feature=${results_dir}/${bm_script}.json + local orig_git_hash + orig_git_hash=$(git rev-parse --verify HEAD) + local output_path_feature=${BASE_OUTPUT_DIR}/${BRANCH_NAME}__${orig_git_hash}.json # Temporarily merge the target branch git checkout -b pr_branch @@ -17,11 +16,9 @@ action() { git merge --no-commit --no-ff origin/"${TARGET_BRANCH}" || (echo "***\nError: There are merge conflicts that need to be resolved.\n***" && false) # create - mkdir -p "$results_dir" + mkdir -p "$(dirname "${output_path_feature}")" - local bm_script="benchmark.py" - - python $bm_script \ + python benchmark.py \ --benchmark_time_unit=ms \ --benchmark_out="${output_path_feature}" \ --benchmark_out_format=json @@ -31,22 +28,23 @@ action() { git stash git checkout origin/"${TARGET_BRANCH}" - local current_git_hash - current_git_hash=$(git rev-parse --verify HEAD) - local results_dir=${BASE_OUTPUT_DIR}/${TARGET_BRANCH}__${current_git_hash} - local output_path_target=${results_dir}/${bm_script}.json + local target_git_hash + target_git_hash=$(git rev-parse --verify HEAD) + local output_path_target=${BASE_OUTPUT_DIR}/${TARGET_BRANCH}__${target_git_hash}.json # create - mkdir -p "$results_dir" - - local bm_script="benchmark.py" + mkdir -p "$(dirname "${output_path_target}")" - python $bm_script \ + python benchmark.py \ --benchmark_time_unit=ms \ --benchmark_out="${output_path_target}" \ --benchmark_out_format=json # Compare both + # first: switch back to original commit + git stash + git checkout "${orig_git_hash}" + python compare.py "${output_path_target}" "${output_path_feature}" } action "$@"