From d78305983bf61ee36c29ca20f303aada7a0cd95e Mon Sep 17 00:00:00 2001 From: leynos Date: Tue, 21 Jul 2026 23:05:27 +0200 Subject: [PATCH] Fan scoped mutation runs out across shards for large change windows (#370) Scheduled (scoped) runs pinned the root target to a single shard regardless of shard-count. When window-hours caught a large batch of changed files that one job outgrew timeout-minutes and was killed, truncating coverage (wildside 308 mutants over 384 min; rstest-bdd cancelled at 42%). Size the scoped root fan-out from the changed-file count: small windows stay single-shard (each shard would re-pay the baseline build for a handful of mutants), while large windows fan out across up to shard-count shards. Every shard carries the same scoped file list and cargo mutants --shard k/N partitions the mutants among them, reusing the existing sharding path with no new caller input. Extra crates stay single-shard, matching full runs. Cover scoped_shard_count and the fan-out matrix with unit tests; update the script, workflow input, and caller-guide docs. --- .github/workflows/mutation-cargo.yml | 7 +- docs/mutation-cargo-workflow.md | 9 +- workflow_scripts/mutation_detect_changes.py | 90 +++++++++++++++---- .../tests/test_mutation_detect_changes.py | 61 +++++++++++++ 4 files changed, 146 insertions(+), 21 deletions(-) diff --git a/.github/workflows/mutation-cargo.yml b/.github/workflows/mutation-cargo.yml index 4935b7c2..360dff24 100644 --- a/.github/workflows/mutation-cargo.yml +++ b/.github/workflows/mutation-cargo.yml @@ -42,8 +42,11 @@ on: default: 300 shard-count: description: >- - Shard fan-out for full (workflow_dispatch) runs; scoped - scheduled runs always use a single shard. + Maximum shard fan-out for the root target. Full + (workflow_dispatch) runs always fan out to this many shards. + Scoped (scheduled) runs stay single-shard for small change + windows and fan out up to this many once the changed-file + count is large, so a wide window cannot outgrow the timeout. type: number default: 6 cargo-mutants-version: diff --git a/docs/mutation-cargo-workflow.md b/docs/mutation-cargo-workflow.md index 56473dcb..adfc5a8a 100644 --- a/docs/mutation-cargo-workflow.md +++ b/docs/mutation-cargo-workflow.md @@ -13,7 +13,12 @@ artefacts. The workflow never gates pull requests. `window-hours` (commit timestamps, so fresh CI clones work), bucket changed `*.rs` files into targets, and mutate only those files. When nothing relevant changed, the run writes a skip message to the job - summary and finishes in seconds. + summary and finishes in seconds. Small windows run the root target as + a single shard; once the changed-file count is large the root fans out + across up to `shard-count` matrix legs (each leg carries the same + scoped file list and `cargo mutants --shard k/N` partitions the + mutants among them), so a wide window cannot outgrow `timeout-minutes` + and silently truncate coverage. - **`workflow_dispatch` runs** bypass the guard and run a full, unscoped mutation of every target, fanning the root target out across `shard-count` matrix legs (`cargo mutants --shard k/N`). Size the @@ -80,7 +85,7 @@ jobs: | `base-ref` | `origin/main` | Reference scanned for changes. | | `timeout-multiplier` | `3` | Per-mutant timeout as a multiple of the baseline. | | `timeout-minutes` | `300` | Per-job ceiling. | -| `shard-count` | `6` | Fan-out for full dispatch runs (scoped runs stay single-shard). | +| `shard-count` | `6` | Maximum root-target fan-out. Full dispatch runs always use it; scoped runs fan out up to it once the change window is large. | | `cargo-mutants-version` | pinned | Tool version; the summary parser is validated against it. | | `extra-args` | (empty) | Extra cargo-mutants arguments (shell-lexed), e.g. `--all-features`. | | `setup-commands` | (empty) | Shell commands run before cargo-mutants in each mutants job (e.g. `sudo apt-get install -y mold` when the repo's `.cargo/config.toml` selects that linker). | diff --git a/workflow_scripts/mutation_detect_changes.py b/workflow_scripts/mutation_detect_changes.py index 93b797de..a853b561 100644 --- a/workflow_scripts/mutation_detect_changes.py +++ b/workflow_scripts/mutation_detect_changes.py @@ -14,8 +14,11 @@ Manual ``workflow_dispatch`` runs bypass the guard entirely and produce a full (unscoped) run, fanned out across ``shard-count`` shards for the root -target. Scoped runs stay single-shard: each shard re-pays the baseline -build-and-test cost, which is not worth it for a handful of mutants. +target. Scoped (scheduled) runs stay single-shard for small windows, where +each shard would needlessly re-pay the baseline build-and-test cost, but +fan the root target out across up to ``shard-count`` shards once the +changed-file count is large, so a wide detection window cannot outgrow the +per-job timeout and truncate coverage (issues #370, #371). Environment Variables --------------------- @@ -33,8 +36,9 @@ INPUT_PATHSPEC : str, optional Git pathspec filtering candidate files. Default: ``*.rs``. INPUT_SHARD_COUNT : int, optional - Number of shards for full (dispatch) runs of the root target. - Default: ``6``. + Maximum shards for the root target. Full (dispatch) runs always fan out + to this many shards; scoped (scheduled) runs fan out up to this many + once the changed-file count is large. Default: ``6``. INPUT_BASE_REF : str, optional Reference whose history is inspected. Default: ``origin/main``. GITHUB_OUTPUT : str @@ -107,7 +111,8 @@ class DetectionConfig: pathspec : str Git pathspec filtering candidate files. shard_count : int - Shard count for full runs of the root target. + Maximum shards for the root target (full runs always use all of + them; large scoped runs fan out up to this many). base_ref : str Reference whose history is inspected. """ @@ -274,22 +279,73 @@ def full_run_matrix(config: DetectionConfig) -> list[MatrixEntry]: return entries +#: Fan a scoped (scheduled) root target out across shards once its +#: changed-file count exceeds this threshold. Small scheduled windows stay +#: single-shard: each shard re-pays the baseline build-and-test cost, not +#: worth it for a handful of mutants. A large window would otherwise run as +#: one job that outgrows ``timeout-minutes`` and silently truncates coverage +#: (issues #370, #371). Sized conservatively so a shard's changed files stay +#: well under the ~200-250-mutant budget observed at the 300-minute ceiling. +SCOPED_FILES_PER_SHARD = 15 + + +def scoped_shard_count(file_count: int, shard_count: int) -> int: + """Return the shard count for a scoped root target. + + Parameters + ---------- + file_count : int + Number of changed files scoped to the root target. + shard_count : int + Upper bound on shards (the ``shard-count`` input); the fan-out never + exceeds it, so callers keep one knob for both full and scoped runs. + + Returns + ------- + int + ``1`` for small windows (at or below + :data:`SCOPED_FILES_PER_SHARD`), otherwise ``ceil(file_count / + SCOPED_FILES_PER_SHARD)`` capped at ``shard_count``. + """ + if file_count <= SCOPED_FILES_PER_SHARD: + return 1 + needed = -(-file_count // SCOPED_FILES_PER_SHARD) + return max(1, min(shard_count, needed)) + + def scoped_run_matrix( buckets: dict[str, list[str]], config: DetectionConfig ) -> list[MatrixEntry]: - """Build the single-shard matrix for a scoped (scheduled) run.""" + """Build the matrix for a scoped (scheduled) run. + + Small windows stay single-shard. When the root target's changed-file + count is large the root fans out across up to ``shard_count`` shards + (``cargo mutants --shard k/N`` partitions the mutant set among the + shards, which all carry the same scoped file list) so no single job + outgrows the timeout budget (issues #370, #371). Extra crates are + assumed small and run as a single shard each, matching + :func:`full_run_matrix`. + """ ordered = sorted(buckets.items(), key=lambda item: (item[0] != ".", item[0])) - del config # scoped runs never shard; kept for signature symmetry - return [ - MatrixEntry( - dir=target_dir, - slug=_slug_for(target_dir), - files=_relative_files(target_dir, files), - shard=0, - shard_count=1, + entries: list[MatrixEntry] = [] + for target_dir, files in ordered: + relative = _relative_files(target_dir, files) + shard_total = ( + scoped_shard_count(len(files), config.shard_count) + if target_dir == "." + else 1 ) - for target_dir, files in ordered - ] + entries.extend( + MatrixEntry( + dir=target_dir, + slug=_slug_for(target_dir), + files=relative, + shard=shard, + shard_count=shard_total, + ) + for shard in range(shard_total) + ) + return entries def matrix_json(entries: list[MatrixEntry]) -> str: @@ -351,7 +407,7 @@ def main( pathspec : str Git pathspec for candidate files. shard_count : int - Shard count for full runs. + Maximum shards for the root target (full and large scoped runs). base_ref : str Reference whose history is inspected. diff --git a/workflow_scripts/tests/test_mutation_detect_changes.py b/workflow_scripts/tests/test_mutation_detect_changes.py index b5bd7d8d..c6cca997 100644 --- a/workflow_scripts/tests/test_mutation_detect_changes.py +++ b/workflow_scripts/tests/test_mutation_detect_changes.py @@ -147,6 +147,67 @@ def test_scoped_run_strips_extra_dir_prefix(self) -> None: assert entries[1].files == "src/lib.rs" assert all(e.shard == 0 and e.shard_count == 1 for e in entries) + @pytest.mark.parametrize( + ("file_count", "shard_count", "expected"), + [ + (0, 6, 1), + (1, 6, 1), + (detect.SCOPED_FILES_PER_SHARD, 6, 1), + (detect.SCOPED_FILES_PER_SHARD + 1, 6, 2), + (detect.SCOPED_FILES_PER_SHARD * 3, 6, 3), + (detect.SCOPED_FILES_PER_SHARD * 100, 6, 6), + (detect.SCOPED_FILES_PER_SHARD * 100, 1, 1), + ], + ) + def test_scoped_shard_count_thresholds( + self, file_count: int, shard_count: int, expected: int + ) -> None: + """Scoped fan-out grows with the batch, capped at ``shard_count``.""" + assert detect.scoped_shard_count(file_count, shard_count) == expected + + def test_small_scoped_root_stays_single_shard(self) -> None: + """A handful of changed files runs the root target as one shard.""" + config = _config(shard_count=6) + files = [f"src/f{i}.rs" for i in range(detect.SCOPED_FILES_PER_SHARD)] + entries = detect.scoped_run_matrix({".": files}, config) + assert len(entries) == 1 + assert entries[0].shard == 0 + assert entries[0].shard_count == 1 + + def test_large_scoped_root_fans_out(self) -> None: + """A large root batch fans out across shards sharing the file list.""" + config = _config(shard_count=6) + files = [f"src/f{i}.rs" for i in range(detect.SCOPED_FILES_PER_SHARD * 3)] + entries = detect.scoped_run_matrix({".": files}, config) + assert len(entries) == 3 + assert [e.shard for e in entries] == [0, 1, 2] + assert all(e.shard_count == 3 for e in entries) + # Every shard carries the same scoped file list; cargo-mutants' + # --shard k/N partitions the mutant set among them. + assert all(e.files == " ".join(files) for e in entries) + assert all(e.slug == "root" for e in entries) + + def test_large_scoped_root_caps_at_shard_count(self) -> None: + """Fan-out never exceeds the configured shard-count ceiling.""" + config = _config(shard_count=4) + files = [f"src/f{i}.rs" for i in range(detect.SCOPED_FILES_PER_SHARD * 20)] + entries = detect.scoped_run_matrix({".": files}, config) + assert len(entries) == 4 + assert all(e.shard_count == 4 for e in entries) + + def test_scoped_extra_crate_stays_single_shard_when_root_fans_out(self) -> None: + """Only the root target fans out; extra crates stay single-shard.""" + config = _config(shard_count=6, extra_crate_dirs=("testkit",)) + root_files = [f"src/f{i}.rs" for i in range(detect.SCOPED_FILES_PER_SHARD * 2)] + buckets = {".": root_files, "testkit": ["testkit/src/lib.rs"]} + entries = detect.scoped_run_matrix(buckets, config) + root = [e for e in entries if e.dir == "."] + extra = [e for e in entries if e.dir == "testkit"] + assert len(root) == 2 + assert len(extra) == 1 + assert extra[0].shard == 0 + assert extra[0].shard_count == 1 + def test_matrix_json_shape(self) -> None: """The matrix output parses back into an ``include`` list.""" entries = detect.full_run_matrix(_config(shard_count=2))