From f47bac0d0443b659b4aa99c6a084cbbf14c527f4 Mon Sep 17 00:00:00 2001 From: Yingfan Guo <115908952+sentomk@users.noreply.github.com> Date: Fri, 31 Jul 2026 04:15:26 +0000 Subject: [PATCH] ci: fix coverage profile collection and move benchmarks to PR checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Coverage fix: gtest_discover_tests runs each TEST() as a separate process. Without LLVM_PROFILE_FILE, all 152 processes wrote to the same default.profraw, each overwriting the previous one — only the last test's coverage survived. Set LLVM_PROFILE_FILE=build/profile/ %p-%m.profraw so each process gets its own file. Benchmark restructure: remove the benchmark job from ci.yml (which runs on push to main, meaning a bench failure put main in a failed state with no easy rollback). Instead: - ci-pr.yml: new benchmark job runs on PRs (informational only, continue-on-error, artifacts only — never blocks merge) - bench-publish.yml: dedicated push-to-main workflow that generates and commits charts to docs/assets/bench/ --- .github/workflows/bench-publish.yml | 88 ++++++++++++++++++++++++++++ .github/workflows/ci-pr.yml | 39 +++++++++++++ .github/workflows/ci.yml | 89 ++++------------------------- 3 files changed, 138 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/bench-publish.yml diff --git a/.github/workflows/bench-publish.yml b/.github/workflows/bench-publish.yml new file mode 100644 index 0000000..19ccd3f --- /dev/null +++ b/.github/workflows/bench-publish.yml @@ -0,0 +1,88 @@ +name: Publish Benchmark Charts + +# Runs on push to main only. Generates benchmark charts and commits +# them to docs/assets/bench/. This is separated from the main CI so +# that benchmark failures never cause the main branch CI to fail. +on: + push: + branches: [main] + paths-ignore: + - "README.md" + - "CONTRIBUTING.md" + - "docs/**" + - "LICENSE" + - ".github/ISSUE_TEMPLATE/**" + +permissions: + contents: write + +concurrency: + group: bench-publish-${{ github.ref }} + cancel-in-progress: true + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.BENCH_BOT_TOKEN || github.token }} + + - name: Configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DPTN_BUILD_TESTS=OFF \ + -DPTN_BUILD_BENCHMARKS=ON \ + -DPTN_SKIP_COMPILER_CHECK=ON + + - name: Build + run: cmake --build build --target ptn_bench ptn_bench_scale --parallel + + - name: Run benchmarks + run: | + mkdir -p bench_results + build/bench/ptn_bench \ + --benchmark_filter="VariantMixed|VariantAltHot|VariantMixedGuarded|VariantAltHotGuarded|ProtocolRouter|CommandParser|PacketMixed|PacketMixedHeavyBind" \ + --benchmark_out=bench_results/ptn_bench.json \ + --benchmark_out_format=json + build/bench/ptn_bench_scale \ + --benchmark_filter="ScaleN" \ + --benchmark_out=bench_results/ptn_bench_scale.json \ + --benchmark_out_format=json + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install plotting dependency + run: python -m pip install matplotlib + + - name: Generate benchmark charts + run: | + python scripts/bench_single_report.py \ + --input bench_results/ptn_bench.json \ + --include "VariantMixed|VariantAltHot|VariantMixedGuarded|VariantAltHotGuarded|ProtocolRouter|CommandParser|PacketMixed|PacketMixedHeavyBind" \ + --outdir docs/assets/bench \ + --prefix latest \ + --title "Patternia vs Standard C++" + python scripts/bench_single_report.py \ + --input bench_results/ptn_bench_scale.json \ + --include "ScaleN" \ + --outdir docs/assets/bench \ + --prefix scale \ + --title "Variant Dispatch Scalability (4-32 alternatives)" + + - name: Commit and push chart update + run: | + if git diff --quiet -- docs/assets/bench/; then + echo "No chart changes." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/assets/bench/ + git commit -m "ci: update benchmark summary chart" + git push diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml index 35af7bb..6639414 100644 --- a/.github/workflows/ci-pr.yml +++ b/.github/workflows/ci-pr.yml @@ -76,3 +76,42 @@ jobs: if: always() run: | echo "Compile-time benchmarks built successfully." + + benchmark: + name: Benchmark (informational) + runs-on: ubuntu-latest + # Informational only — does not block PR merge. + # Benchmark failures surface as artifacts for review, not as red X. + continue-on-error: true + + steps: + - uses: actions/checkout@v4 + + - name: Configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DPTN_BUILD_TESTS=OFF \ + -DPTN_BUILD_BENCHMARKS=ON \ + -DPTN_SKIP_COMPILER_CHECK=ON + + - name: Build + run: cmake --build build --target ptn_bench ptn_bench_scale --parallel + + - name: Run benchmarks + run: | + mkdir -p bench_results + build/bench/ptn_bench \ + --benchmark_filter="VariantMixed|VariantAltHot|VariantMixedGuarded|VariantAltHotGuarded|ProtocolRouter|CommandParser|PacketMixed|PacketMixedHeavyBind" \ + --benchmark_out=bench_results/ptn_bench.json \ + --benchmark_out_format=json + build/bench/ptn_bench_scale \ + --benchmark_filter="ScaleN" \ + --benchmark_out=bench_results/ptn_bench_scale.json \ + --benchmark_out_format=json + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-${{ github.event.pull_request.number || github.sha }} + path: bench_results/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a914494..5b3748f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,12 +79,20 @@ jobs: run: cmake --build build --parallel - name: Run Tests - run: ctest --test-dir build --output-on-failure + env: + # Give each test process its own raw profile file. + # %p = PID, %m = binary module name → avoids overwrites across + # the 150+ gtest_discover_tests processes. + LLVM_PROFILE_FILE: ${{ github.workspace }}/build/profile/%p-%m.profraw + run: | + mkdir -p build/profile + ctest --test-dir build --output-on-failure - name: Capture coverage run: | - # Merge raw profiles from all test runs. - find build -name '*.profraw' > profraw_list.txt + # Merge raw profiles from all test processes. + find build/profile -name '*.profraw' > profraw_list.txt + echo "Found $(wc -l < profraw_list.txt) raw profiles" llvm-profdata-18 merge \ -input-files profraw_list.txt \ -output coverage.profdata \ @@ -111,78 +119,3 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.info fail_ci_if_error: false - - benchmark: - needs: build-and-test - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.BENCH_BOT_TOKEN || github.token }} - - - name: Configure - run: | - cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DPTN_BUILD_TESTS=OFF \ - -DPTN_BUILD_BENCHMARKS=ON \ - -DPTN_SKIP_COMPILER_CHECK=ON - - - name: Build - run: cmake --build build --target ptn_bench ptn_bench_scale --parallel - - - name: Run benchmarks - run: | - mkdir -p bench_results - build/bench/ptn_bench \ - --benchmark_filter="VariantMixed|VariantAltHot|VariantMixedGuarded|VariantAltHotGuarded|ProtocolRouter|CommandParser|PacketMixed|PacketMixedHeavyBind" \ - --benchmark_out=bench_results/ptn_bench.json \ - --benchmark_out_format=json - build/bench/ptn_bench_scale \ - --benchmark_filter="ScaleN" \ - --benchmark_out=bench_results/ptn_bench_scale.json \ - --benchmark_out_format=json - - - name: Upload benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark-results-${{ github.sha }} - path: bench_results/ - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - - name: Install plotting dependency - run: python -m pip install matplotlib - - - name: Generate benchmark chart - run: | - python scripts/bench_single_report.py \ - --input bench_results/ptn_bench.json \ - --include "VariantMixed|VariantAltHot|VariantMixedGuarded|VariantAltHotGuarded|ProtocolRouter|CommandParser|PacketMixed|PacketMixedHeavyBind" \ - --outdir docs/assets/bench \ - --prefix latest \ - --title "Patternia vs Standard C++" - python scripts/bench_single_report.py \ - --input bench_results/ptn_bench_scale.json \ - --include "ScaleN" \ - --outdir docs/assets/bench \ - --prefix scale \ - --title "Variant Dispatch Scalability (4-32 alternatives)" - - - name: Commit and push chart update - run: | - if git diff --quiet -- docs/assets/bench/; then - echo "No chart changes." - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add docs/assets/bench/ - git commit -m "ci: update benchmark summary chart" - git push