From 47c06f9282fc8cd1d0364f5e77f8085829068327 Mon Sep 17 00:00:00 2001 From: MatthewMckee4 Date: Wed, 15 Apr 2026 16:13:56 +0100 Subject: [PATCH 1/4] Switch CodSpeed benchmarks to instrumented mode --- .github/actionlint.yaml | 5 ----- .github/workflows/ci.yml | 9 ++++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 3c69aef3..b0de1ec3 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,7 +1,2 @@ # Configuration for the actionlint tool, which we run via prek # to verify the correctness of the syntax in our GitHub Actions workflows. - -self-hosted-runner: - # Various runners we use that aren't recognized out-of-the-box by actionlint: - labels: - - codspeed-macro diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7917a81c..63edf67a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,10 +243,10 @@ jobs: - name: Build docs run: uv run --isolated --only-group docs zensical build - benchmarks-walltime: - name: "walltime benchmarks" + benchmarks-instrumented: + name: "instrumented benchmarks" - runs-on: ubuntu-latest # should be codspeed-macro + runs-on: ubuntu-latest needs: determine_changes if: ${{ (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }} @@ -283,7 +283,7 @@ jobs: - name: Build benchmarks env: PYO3_PYTHON: ${{ steps.setup-python.outputs.python-path }} - run: cargo codspeed build -m walltime + run: cargo codspeed build - name: Run benchmarks uses: CodSpeedHQ/action@972e3437949c89e1357ebd1a2dbc852fcbc57245 # v4.5.1 @@ -292,4 +292,3 @@ jobs: with: token: ${{ secrets.CODSPEED_TOKEN }} run: cargo codspeed run --bench karva_benchmark_walltime - mode: walltime From 18be4c554b03ba888f1a4ca8a9637b8e98be5fe0 Mon Sep 17 00:00:00 2001 From: MatthewMckee4 Date: Wed, 15 Apr 2026 16:16:12 +0100 Subject: [PATCH 2/4] Pin benchmark workers to 2 for consistency --- Cargo.lock | 1 - crates/karva_benchmark/Cargo.toml | 1 - crates/karva_benchmark/src/lib.rs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69f80a82..e90653a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1108,7 +1108,6 @@ dependencies = [ "karva_metadata", "karva_project", "karva_runner", - "karva_static", "ruff_python_ast", "serde", "serde_json", diff --git a/crates/karva_benchmark/Cargo.toml b/crates/karva_benchmark/Cargo.toml index 33d8ea0c..5b745726 100644 --- a/crates/karva_benchmark/Cargo.toml +++ b/crates/karva_benchmark/Cargo.toml @@ -20,7 +20,6 @@ karva_logging = { workspace = true } karva_metadata = { workspace = true } karva_project = { workspace = true } karva_runner = { workspace = true } -karva_static = { workspace = true } ruff_python_ast = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/karva_benchmark/src/lib.rs b/crates/karva_benchmark/src/lib.rs index a4038d9e..e26f87c1 100644 --- a/crates/karva_benchmark/src/lib.rs +++ b/crates/karva_benchmark/src/lib.rs @@ -36,7 +36,7 @@ pub fn setup_project() -> Project { /// Run karva tests against the prepared project once. pub fn run_karva(project: &Project) { let config = karva_runner::ParallelTestConfig { - num_workers: karva_static::max_parallelism().get(), + num_workers: 2, no_cache: false, create_ctrlc_handler: false, last_failed: false, From 7cfeb36c142e6815724af324674039ed0592e8c1 Mon Sep 17 00:00:00 2001 From: MatthewMckee4 Date: Wed, 15 Apr 2026 16:16:58 +0100 Subject: [PATCH 3/4] Rename karva_benchmark_walltime bench to karva_benchmark --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 2 +- crates/karva_benchmark/Cargo.toml | 2 +- .../benches/{karva_benchmark_walltime.rs => karva_benchmark.rs} | 0 crates/karva_benchmark/src/lib.rs | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename crates/karva_benchmark/benches/{karva_benchmark_walltime.rs => karva_benchmark.rs} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63edf67a..383ddf14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -291,4 +291,4 @@ jobs: PYO3_PYTHON: ${{ steps.setup-python.outputs.python-path }} with: token: ${{ secrets.CODSPEED_TOKEN }} - run: cargo codspeed run --bench karva_benchmark_walltime + run: cargo codspeed run --bench karva_benchmark diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa0e215b..e4cd871c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ Infrastructure and tooling: - `karva_python` — the PyO3 `cdylib` that produces the Python wheel and wraps both `karva` and `karva_worker`. - `karva_macros` — procedural macros. - `karva_dev` — dev tools such as CLI reference generation. -- `karva_benchmark` — wall-time benchmark that runs `karva test` against a pinned snapshot of `karva-benchmark-1`. +- `karva_benchmark` — benchmark that runs `karva test` against a pinned snapshot of `karva-benchmark-1`. ### Prerequisites diff --git a/crates/karva_benchmark/Cargo.toml b/crates/karva_benchmark/Cargo.toml index 5b745726..4cbf6d27 100644 --- a/crates/karva_benchmark/Cargo.toml +++ b/crates/karva_benchmark/Cargo.toml @@ -25,7 +25,7 @@ serde = { workspace = true } serde_json = { workspace = true } [[bench]] -name = "karva_benchmark_walltime" +name = "karva_benchmark" harness = false [lints] diff --git a/crates/karva_benchmark/benches/karva_benchmark_walltime.rs b/crates/karva_benchmark/benches/karva_benchmark.rs similarity index 100% rename from crates/karva_benchmark/benches/karva_benchmark_walltime.rs rename to crates/karva_benchmark/benches/karva_benchmark.rs diff --git a/crates/karva_benchmark/src/lib.rs b/crates/karva_benchmark/src/lib.rs index e26f87c1..5ea8c32a 100644 --- a/crates/karva_benchmark/src/lib.rs +++ b/crates/karva_benchmark/src/lib.rs @@ -1,4 +1,4 @@ -//! Wall-time benchmark for karva. +//! Benchmark for karva. //! //! Clones a fixed snapshot of //! into `target/benchmark_cache/`, installs its dependencies via uv, and runs From 4376e524c85d9ef98d067881716025a23ed421d5 Mon Sep 17 00:00:00 2001 From: MatthewMckee4 Date: Wed, 15 Apr 2026 16:21:22 +0100 Subject: [PATCH 4/4] Set CodSpeed mode to simulation (required in v4) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 383ddf14..8f695238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -292,3 +292,4 @@ jobs: with: token: ${{ secrets.CODSPEED_TOKEN }} run: cargo codspeed run --bench karva_benchmark + mode: simulation