diff --git a/docs/developers-guide.md b/docs/developers-guide.md index f5db883..123183d 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -79,6 +79,13 @@ The main `.github/workflows/ci.yml` workflow runs ordinary `make test` without `WITH_ACT=1` so the slower container-backed checks run in parallel instead of blocking the main test and coverage path. +The template also renders `.github/workflows/mutation-testing.yml` into +generated projects; its rendered contract is asserted by +`_assert_mutation_workflow_contracts` in +`tests/helpers/tooling_contracts/workflows.py`. Dependabot owns the pinned +reusable-workflow commit SHA, so the contract test asserts the pin format (a +full commit SHA) rather than a hard-coded value. + ## Required Tooling The parent tests expect these tools to be available when validating generated diff --git a/docs/users-guide.md b/docs/users-guide.md index 0e46957..ef38aa3 100644 --- a/docs/users-guide.md +++ b/docs/users-guide.md @@ -69,3 +69,36 @@ The generated `Makefile` exposes these public targets: Install `clang`, `lld`, `mold`, `python3`, and `cargo-audit` before running the full generated workflow locally on Linux. + +## Scheduled Mutation Testing + +Generated projects include `.github/workflows/mutation-testing.yml`, a scheduled +GitHub Actions workflow that runs mutation testing with `cargo-mutants`. It is a +thin caller of the shared `leynos/shared-actions` `mutation-cargo` reusable +workflow. + +Mutation testing measures test-suite quality. It introduces small changes +(mutants) into the source and confirms the tests fail in response. A surviving +mutant marks a code path the tests do not meaningfully exercise, so promote it +into a new test rather than ignoring it. + +The workflow runs on a daily schedule (09:15 UTC by default) and can also be +started manually from the **Actions** tab with **Run workflow**. Scheduled runs +mutate only files changed within the detection window, so routine runs stay +fast; a manual dispatch mutates the whole crate, fanned out across shards. +Because the runs are scheduled rather than gating pull requests, they surface +coverage gaps without slowing day-to-day CI, and they do not block merges. + +When adopting the workflow in a new repository, stagger the cron slot: pick an +unclaimed daily time to avoid concurrent runs across related repositories. The +`mutation` job runs with a least-privilege token (`contents: read` plus +`id-token: write` for workflow-source resolution). + +Results land in the run's job summary: a final job posts per-target outcome +counts and a table of surviving mutants, and each shard uploads its +`mutants.out/` directory as a `mutation-report-*` artefact. When nothing +relevant changed, the run writes a skip message and finishes in seconds. +Surviving mutants and timeouts are informational and leave the run green, so a +red run means something actually broke — a usage error, an already-failing test +baseline, or an internal error. Watch for those through GitHub's notifications +for failed scheduled runs. diff --git a/template/.github/workflows/mutation-testing.yml b/template/.github/workflows/mutation-testing.yml new file mode 100644 index 0000000..bc49243 --- /dev/null +++ b/template/.github/workflows/mutation-testing.yml @@ -0,0 +1,47 @@ +name: Mutation testing + +# Thin caller of the shared mutation-testing reusable workflow; caller +# guide: leynos/shared-actions docs/mutation-cargo-workflow.md. +# Scheduled runs mutate only files changed within the detection window; +# manual dispatch runs mutate everything, fanned out across shards. To +# test a branch, select it in the Actions "Run workflow" control. + +'on': + schedule: + # Daily, 09:15 UTC. Stagger this slot when adopting the workflow in a + # new repository: the estate already occupies 03:05-09:05 UTC, so pick + # an unclaimed slot to avoid concurrent runs across repositories. + - cron: "15 9 * * *" + workflow_dispatch: + +# Default the token to no scopes; the job opts in explicitly. +permissions: {} + +# Serialize runs per ref: a dispatch during a scheduled run (or vice +# versa) queues rather than racing. Runs are informational, so queued +# runs wait instead of cancelling. +concurrency: + group: mutation-testing-${{ github.ref }} + cancel-in-progress: false + +jobs: + mutation: + permissions: + contents: read + id-token: write # OIDC workflow-source resolution + uses: leynos/shared-actions/.github/workflows/mutation-cargo.yml@0c8bd605f8966a004decb0b54f717a745f51bc57 + with: + # Match the CI baseline (`make test` runs --all-targets + # --all-features) so the test command exercised against mutants + # mirrors what gates pull requests. + extra-args: "--all-features" + # .cargo/config.toml links Linux dev targets with clang and + # -fuse-ld=mold, so the mutants jobs need the same packages that + # ci.yml installs before any cargo command runs on a bare + # ubuntu-latest runner. The reusable workflow executes these under + # bash with `set -euo pipefail`. + setup-commands: | + export DEBIAN_FRONTEND=noninteractive + # mold accelerates dev builds; coverage uses lld for llvm-tools compatibility. + sudo apt-get update \ + && sudo apt-get install --yes --no-install-recommends clang lld mold diff --git a/template/docs/developers-guide.md.jinja b/template/docs/developers-guide.md.jinja index bc0fb8b..2056c55 100644 --- a/template/docs/developers-guide.md.jinja +++ b/template/docs/developers-guide.md.jinja @@ -25,6 +25,13 @@ The main `.github/workflows/ci.yml` workflow deliberately does not run `make test WITH_ACT=1`; the separate Act workflow runs those slower container-backed checks in parallel. +A scheduled `.github/workflows/mutation-testing.yml` workflow also runs +`cargo-mutants` via the shared reusable workflow, daily and on manual +dispatch. It is informational and does not gate pull requests. Dependabot +keeps its pinned reusable-workflow SHA current. See the user guide's +"Scheduled Mutation Testing" section for behaviour, and promote surviving +mutants into new tests. + ## Tooling Development builds use Cranelift for debug code generation. On Linux targets, diff --git a/template/docs/repository-layout.md.jinja b/template/docs/repository-layout.md.jinja index 6ecb2e0..2cc430b 100644 --- a/template/docs/repository-layout.md.jinja +++ b/template/docs/repository-layout.md.jinja @@ -18,6 +18,7 @@ compact and omits build output such as `target/`. │ └── workflows/ │ ├── act-validation.yml │ ├── ci.yml +│ ├── mutation-testing.yml {% if flavour == 'app' %} │ └── release.yml {% endif %} @@ -55,6 +56,8 @@ compact and omits build output such as `target/`. validation through `act` separately from main CI. - `.github/workflows/ci.yml`: Runs the generated project's continuous integration checks. +- `.github/workflows/mutation-testing.yml`: Runs scheduled mutation testing + through the shared `mutation-cargo` reusable workflow. {% if flavour == 'app' %} - `.github/workflows/release.yml`: Builds and publishes binary release artefacts for the application flavour. diff --git a/template/docs/users-guide.md.jinja b/template/docs/users-guide.md.jinja index 914e7ac..5e06f5f 100644 --- a/template/docs/users-guide.md.jinja +++ b/template/docs/users-guide.md.jinja @@ -44,3 +44,36 @@ The generated `Makefile` exposes these public targets: Install `clang`, `lld`, `mold`, `python3`, and `cargo-audit` before running the full generated workflow locally on Linux. + +## Scheduled Mutation Testing + +Generated projects include `.github/workflows/mutation-testing.yml`, a scheduled +GitHub Actions workflow that runs mutation testing with `cargo-mutants`. It is a +thin caller of the shared `leynos/shared-actions` `mutation-cargo` reusable +workflow. + +Mutation testing measures test-suite quality. It introduces small changes +(mutants) into the source and confirms the tests fail in response. A surviving +mutant marks a code path the tests do not meaningfully exercise, so promote it +into a new test rather than ignoring it. + +The workflow runs on a daily schedule (09:15 UTC by default) and can also be +started manually from the **Actions** tab with **Run workflow**. Scheduled runs +mutate only files changed within the detection window, so routine runs stay +fast; a manual dispatch mutates the whole crate, fanned out across shards. +Because the runs are scheduled rather than gating pull requests, they surface +coverage gaps without slowing day-to-day CI, and they do not block merges. + +When adopting the workflow in a new repository, stagger the cron slot: pick an +unclaimed daily time to avoid concurrent runs across related repositories. The +`mutation` job runs with a least-privilege token (`contents: read` plus +`id-token: write` for workflow-source resolution). + +Results land in the run's job summary: a final job posts per-target outcome +counts and a table of surviving mutants, and each shard uploads its +`mutants.out/` directory as a `mutation-report-*` artefact. When nothing +relevant changed, the run writes a skip message and finishes in seconds. +Surviving mutants and timeouts are informational and leave the run green, so a +red run means something actually broke — a usage error, an already-failing test +baseline, or an internal error. Watch for those through GitHub's notifications +for failed scheduled runs. diff --git a/tests/helpers/tooling_contracts/mutation.py b/tests/helpers/tooling_contracts/mutation.py new file mode 100644 index 0000000..c5c604d --- /dev/null +++ b/tests/helpers/tooling_contracts/mutation.py @@ -0,0 +1,135 @@ +"""Assert the rendered mutation-testing workflow contract. + +The mutation job's pinned reusable-workflow SHA is asserted by shape, not by +exact value: Dependabot owns bumping it, so hard-coding a value here would make +the suite fail on every routine bump. +""" + +from __future__ import annotations + +import re +import shlex + +from tests.helpers.generated_files import ( + parse_yaml_mapping, + require_mapping, + require_sequence, +) + +_MUTATION_JOB_PERMISSIONS = {"contents": "read", "id-token": "write"} +_MUTATION_SETUP_PACKAGES = ("clang", "lld", "mold") +_MUTATION_REUSABLE_WORKFLOW = ( + "leynos/shared-actions/.github/workflows/mutation-cargo.yml" +) +_MUTATION_CRON = "15 9 * * *" +_MUTATION_CONCURRENCY_GROUP = "mutation-testing-${{ github.ref }}" + +_SHELL_COMMAND_SEPARATORS = frozenset({"&&", "||", ";", "|", "&"}) + + +def _split_shell_commands(tokens: list[str]) -> list[list[str]]: + """Split a shell token stream into commands on shell control operators.""" + commands: list[list[str]] = [] + current: list[str] = [] + for token in tokens: + if token in _SHELL_COMMAND_SEPARATORS: + commands.append(current) + current = [] + else: + current.append(token) + commands.append(current) + return commands + + +def _apt_install_packages(command: list[str]) -> list[str] | None: + """Return non-flag args of ``[sudo] apt-get install ...``, else ``None``.""" + words = command[1:] if command[:1] == ["sudo"] else command + if words[:2] != ["apt-get", "install"]: + return None + return [arg for arg in words[2:] if not arg.startswith("-")] + + +def _extract_apt_install_packages(setup_commands: str) -> list[str]: + """Return the packages installed by a setup-commands ``apt-get install``.""" + # Join backslash continuations so each command sits on one logical line. + joined = setup_commands.replace("\\\n", " ") + for line in joined.splitlines(): + # Surface malformed shell instead of masking it as a valid install; a + # setup-commands script that cannot be parsed is not runnable. + try: + tokens = shlex.split(line, comments=True) + except ValueError as error: + raise AssertionError( + "expected mutation job setup-commands to be parseable shell, " + f"got {line!r}" + ) from error + # Only a real `[sudo] apt-get install` command counts; embedded text + # such as an echo argument must never be treated as an install. + for command in _split_shell_commands(tokens): + packages = _apt_install_packages(command) + if packages is not None: + return packages + return [] + + +def _assert_mutation_workflow_contracts(mutation_workflow: str) -> None: + """Assert generated mutation-testing workflow contracts.""" + parsed = parse_yaml_mapping(mutation_workflow, "mutation-testing workflow") + assert parsed.get("permissions") == {}, ( + "expected mutation-testing workflow root permissions to grant no scopes" + ) + triggers = require_mapping(parsed, "on", "mutation-testing workflow") + assert set(triggers) == {"schedule", "workflow_dispatch"}, ( + "expected mutation-testing workflow to trigger only on schedule and " + "workflow_dispatch, rejecting push or pull-request runs" + ) + schedule = require_sequence(triggers, "schedule", "mutation-testing workflow on") + assert schedule == [{"cron": _MUTATION_CRON}], ( + "expected mutation-testing workflow on.schedule to be exactly one cron " + f"entry {_MUTATION_CRON!r}" + ) + assert "workflow_dispatch" in triggers, ( + "expected mutation-testing workflow to support manual workflow_dispatch" + ) + concurrency = require_mapping(parsed, "concurrency", "mutation-testing workflow") + assert concurrency.get("group") == _MUTATION_CONCURRENCY_GROUP, ( + "expected mutation-testing workflow concurrency group " + f"{_MUTATION_CONCURRENCY_GROUP!r}" + ) + assert concurrency.get("cancel-in-progress") is False, ( + "expected mutation-testing workflow to queue runs " + "(concurrency.cancel-in-progress: false)" + ) + jobs = require_mapping(parsed, "jobs", "mutation-testing workflow") + mutation = require_mapping(jobs, "mutation", "mutation-testing workflow jobs") + mutation_uses = str(mutation.get("uses", "")) + pinned_prefix = f"{_MUTATION_REUSABLE_WORKFLOW}@" + assert mutation_uses.startswith(pinned_prefix), ( + "expected mutation job to call the shared mutation-cargo reusable workflow" + ) + # The specific SHA is owned by Dependabot and deliberately not hard-coded, so + # the contract survives dependency bumps while requiring an immutable pin. + assert re.fullmatch(r"[0-9a-f]{40}", mutation_uses[len(pinned_prefix) :]), ( + "expected mutation job to pin the reusable workflow to a full commit SHA" + ) + permissions = require_mapping(mutation, "permissions", "mutation job") + assert permissions == _MUTATION_JOB_PERMISSIONS, ( + "expected mutation job permissions to stay scoped to " + "contents: read and id-token: write" + ) + inputs = require_mapping(mutation, "with", "mutation job") + assert inputs.get("extra-args") == "--all-features", ( + "expected mutation job to mirror the CI --all-features test baseline" + ) + setup_commands = inputs.get("setup-commands") + assert isinstance(setup_commands, str), ( + "expected mutation job setup-commands to be a string" + ) + installed_packages = _extract_apt_install_packages(setup_commands) + assert installed_packages, ( + "expected mutation job setup-commands to install packages via apt-get" + ) + for package in _MUTATION_SETUP_PACKAGES: + assert package in installed_packages, ( + f"expected mutation job setup-commands apt-get install to include {package}" + ) diff --git a/tests/helpers/tooling_contracts/orchestration.py b/tests/helpers/tooling_contracts/orchestration.py index 10b20c4..e42a4fa 100644 --- a/tests/helpers/tooling_contracts/orchestration.py +++ b/tests/helpers/tooling_contracts/orchestration.py @@ -12,6 +12,9 @@ assert_documentation_navigation_contracts, ) from tests.helpers.tooling_contracts.makefile import _assert_makefile_contracts +from tests.helpers.tooling_contracts.mutation import ( + _assert_mutation_workflow_contracts, +) from tests.helpers.tooling_contracts.workflows import ( _assert_audit_workflow_contracts, _assert_ci_workflow_contracts, @@ -32,6 +35,7 @@ def assert_generated_tooling_contracts( ci_workflow: str, audit_workflow: str, act_workflow: str, + mutation_workflow: str, docs_contents: str, repository_layout: str, readme: str, @@ -64,6 +68,8 @@ def assert_generated_tooling_contracts( Rendered generated scheduled dependency audit workflow text. act_workflow Rendered generated act-validation workflow text. + mutation_workflow + Rendered generated mutation-testing workflow text. docs_contents Rendered ``docs/contents.md`` text. repository_layout @@ -99,6 +105,7 @@ def assert_generated_tooling_contracts( parsed_ci_workflow, ci_workflow, act_workflow, test_stub ) _assert_audit_workflow_contracts(audit_workflow) + _assert_mutation_workflow_contracts(mutation_workflow) assert_documentation_navigation_contracts(docs_contents, repository_layout, flavour) assert "[Documentation contents](docs/contents.md)" in readme, ( "expected generated README to link to the documentation contents" diff --git a/tests/helpers/tooling_contracts/workflows.py b/tests/helpers/tooling_contracts/workflows.py index cf04d73..8ee5cdf 100644 --- a/tests/helpers/tooling_contracts/workflows.py +++ b/tests/helpers/tooling_contracts/workflows.py @@ -23,9 +23,11 @@ _UPLOAD_CODESCENE_COVERAGE_USES_RE = re.compile( r"^leynos/shared-actions/\.github/actions/upload-codescene-coverage@[0-9a-f]{40}$" ) -_SETUP_RUST_USES_TEXT_RE = re.compile( +_SETUP_RUST_USES_RE = re.compile( r"leynos/shared-actions/\.github/actions/setup-rust@[0-9a-f]{40}" ) +_SETUP_PYTHON_USES_RE = re.compile(r"actions/setup-python@[0-9a-f]{40}") +_UPLOAD_ARTIFACT_USES_RE = re.compile(r"actions/upload-artifact@[0-9a-f]{40}") def _is_pinned_action(uses: str, action_path: str) -> bool: @@ -183,7 +185,7 @@ def _assert_audit_workflow_contracts(audit_workflow: str) -> None: "expected generated audit workflow to include one Setup Rust step" ) setup_rust_uses = str(setup_rust_steps[0].get("uses", "")) - assert _SETUP_RUST_USES_TEXT_RE.fullmatch(setup_rust_uses), ( + assert _SETUP_RUST_USES_RE.fullmatch(setup_rust_uses), ( "expected generated audit workflow to use setup-rust pinned to a full " f"40-hex commit SHA, got {setup_rust_uses!r}" ) @@ -356,9 +358,8 @@ def _assert_ci_workflow_contracts( assert "Cache Whitaker installation" in ci_workflow, ( "expected generated CI workflow to cache Whitaker installation" ) - assert _SETUP_RUST_USES_TEXT_RE.search(ci_workflow), ( - "expected generated CI workflow to use the shared setup-rust action " - "pinned to a full 40-hex commit SHA" + _assert_pinned_step_uses( + steps, _SETUP_RUST_USES_RE, "generated CI shared setup-rust action" ) build_test_checkout = [ step @@ -398,9 +399,9 @@ def _assert_ci_workflow_contracts( assert "Setup Python for audit manifest extraction" in ci_workflow, ( "expected generated CI workflow to install Python for audit metadata parsing" ) - assert "actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405" in ( - ci_workflow - ), "expected generated CI workflow to pin setup-python for audit parsing" + _assert_pinned_step_uses( + steps, _SETUP_PYTHON_USES_RE, "generated CI setup-python action" + ) assert "make audit" in ci_workflow, ( "expected generated CI workflow to run the dependency audit gate" ) @@ -453,10 +454,11 @@ def _assert_release_workflow_contracts(release_workflow: str) -> None: step.get("with", {}).get("persist-credentials") is False for step in release_checkout_steps ), "expected release workflow checkout steps to disable credentials" - assert ( - "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" - in release_workflow - ), "expected app release workflow to use pinned upload-artifact action" + _assert_pinned_step_uses( + _iter_job_steps(jobs), + _UPLOAD_ARTIFACT_USES_RE, + "app release upload-artifact action", + ) cross_revision = "88f49ff79e777bef6d3564531636ee4d3cc2f8d2" assert f"CROSS_REVISION: {cross_revision}" in release_workflow, ( "expected app release workflow to pin cross to an immutable revision" @@ -500,3 +502,23 @@ def extract_checkout_steps(jobs: dict[str, Any]) -> list[dict[str, Any]]: if isinstance(step, dict) and str(step.get("uses", "")).startswith("actions/checkout@") ] + + +def _iter_job_steps(jobs: dict[str, Any]) -> list[Any]: + """Return every step across all jobs in a parsed jobs mapping.""" + return [ + step + for job in jobs.values() + if isinstance(job, dict) + for step in job.get("steps", []) + ] + + +def _assert_pinned_step_uses( + steps: list[Any], uses_re: "re.Pattern[str]", label: str +) -> None: + """Assert one workflow step's ``uses:`` fully matches ``uses_re``.""" + assert any( + isinstance(step, dict) and uses_re.fullmatch(str(step.get("uses", ""))) + for step in steps + ), f"expected {label} pinned to a full 40-hex commit SHA in a workflow step" diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 9e666db..c9afe83 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -169,35 +169,39 @@ def test_require_sequence_property(key: str, value: object) -> None: _full_shas = st.text(alphabet=_HEX_ALPHABET, min_size=40, max_size=40) +# Focused strategies for refs that are never a 40-character lowercase-hex SHA, +# composed into ``_non_sha_refs`` below. +_too_short_hex_refs: st.SearchStrategy[str] = st.text( + alphabet=_HEX_ALPHABET, min_size=0, max_size=39 +) +_too_long_hex_refs: st.SearchStrategy[str] = st.text( + alphabet=_HEX_ALPHABET, min_size=41, max_size=64 +) +_branch_refs: st.SearchStrategy[str] = st.sampled_from( + ("main", "master", "rolling", "HEAD", "v1.2.3") +) + + @st.composite -def _non_sha_refs(draw: st.DrawFn) -> str: - """Draw refs that are never a 40-character lowercase-hex commit SHA.""" - kind = draw(st.sampled_from(("short", "long", "uppercased", "branch"))) - match kind: - case "short": - size = draw(st.integers(min_value=0, max_value=39)) - return draw(st.text(alphabet=_HEX_ALPHABET, min_size=size, max_size=size)) - case "long": - size = draw(st.integers(min_value=41, max_value=64)) - return draw(st.text(alphabet=_HEX_ALPHABET, min_size=size, max_size=size)) - case "uppercased": - hexes = draw(st.text(alphabet=_HEX_ALPHABET, min_size=40, max_size=40)) - position = draw(st.integers(min_value=0, max_value=39)) - existing = hexes[position] - # Uppercasing a hex digit is a no-op, so fall back to a hex letter - # to keep the ref out of the lowercase-hex SHA shape. - flipped = ( - existing.upper() - if existing.isalpha() - else draw(st.sampled_from("ABCDEF")) - ) - return f"{hexes[:position]}{flipped}{hexes[position + 1 :]}" - case "branch": - return draw( - st.sampled_from(("main", "master", "rolling", "HEAD", "v1.2.3")) - ) - case _: # pragma: no cover - kind is drawn from the sampled set above - raise AssertionError(f"unexpected ref kind: {kind}") +def _uppercased_hex_refs(draw: st.DrawFn) -> str: + """Draw a 40-character hex ref carrying at least one uppercase digit.""" + hexes = draw(st.text(alphabet=_HEX_ALPHABET, min_size=40, max_size=40)) + position = draw(st.integers(min_value=0, max_value=39)) + existing = hexes[position] + # Uppercasing a hex digit is a no-op, so fall back to a hex letter to keep + # the ref out of the lowercase-hex SHA shape. + flipped = ( + existing.upper() if existing.isalpha() else draw(st.sampled_from("ABCDEF")) + ) + return f"{hexes[:position]}{flipped}{hexes[position + 1 :]}" + + +_non_sha_refs: st.SearchStrategy[str] = st.one_of( + _too_short_hex_refs, + _too_long_hex_refs, + _uppercased_hex_refs(), + _branch_refs, +) @given(path=_action_paths, sha=_full_shas) @@ -211,7 +215,7 @@ def test_is_pinned_action_accepts_full_sha(path: str, sha: str) -> None: ) -@given(path=_action_paths, ref=_non_sha_refs()) +@given(path=_action_paths, ref=_non_sha_refs) def test_is_pinned_action_rejects_non_sha_refs(path: str, ref: str) -> None: """Refs that are not a full 40-hex commit SHA are never treated as pinned.""" assert not _is_pinned_action(f"{path}@{ref}", path), ( @@ -578,6 +582,6 @@ def _ci_workflow(*, persist_credentials: str, coverage_inputs: str) -> str: with: persist-credentials: {persist_credentials} - name: Test and Measure Coverage - uses: leynos/shared-actions/.github/actions/generate-coverage@927edd45ae77be4251a8a18ca9eb5613a2e32cbd + uses: leynos/shared-actions/.github/actions/generate-coverage@0000000000000000000000000000000000000000 with: {coverage_inputs}""" diff --git a/tests/test_mutation_contracts.py b/tests/test_mutation_contracts.py new file mode 100644 index 0000000..6c48d28 --- /dev/null +++ b/tests/test_mutation_contracts.py @@ -0,0 +1,145 @@ +"""Validate the mutation-testing workflow contract helpers.""" + +from __future__ import annotations + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from tests.helpers.tooling_contracts.mutation import ( + _MUTATION_CRON, + _assert_mutation_workflow_contracts, + _extract_apt_install_packages, +) + + +def test_extract_apt_install_packages_returns_non_flag_arguments() -> None: + """Return install arguments across backslash continuations, dropping flags.""" + setup_commands = ( + "export DEBIAN_FRONTEND=noninteractive\n" + "sudo apt-get update \\\n" + " && sudo apt-get install --yes --no-install-recommends clang lld mold\n" + ) + + assert _extract_apt_install_packages(setup_commands) == ["clang", "lld", "mold"], ( + "expected only the non-flag install arguments to be returned" + ) + + +def test_extract_apt_install_packages_ignores_commented_installs() -> None: + """Skip commented-out install commands so they cannot satisfy the contract.""" + setup_commands = "# sudo apt-get install --yes clang lld mold\necho skip\n" + + assert _extract_apt_install_packages(setup_commands) == [], ( + "expected a commented-out apt-get install to be ignored" + ) + + +def test_extract_apt_install_packages_rejects_malformed_shell() -> None: + """Surface malformed setup-commands shell instead of masking it as valid.""" + setup_commands = 'sudo apt-get install --yes "clang\n' + + with pytest.raises(AssertionError, match="parseable shell"): + _extract_apt_install_packages(setup_commands) + + +def test_extract_apt_install_packages_ignores_echoed_marker() -> None: + """Ignore an echo whose argument merely contains the install marker text.""" + setup_commands = 'echo "run apt-get install clang lld mold first"\n' + + assert _extract_apt_install_packages(setup_commands) == [], ( + "expected embedded marker text in an echo argument not to count as an " + "install command" + ) + + +# Shell-safe package tokens: a leading alphanumeric (so they never look like a +# flag) followed by characters valid in a Debian package name. Flag tokens +# always begin with ``--`` so the parser drops them. +_apt_package_tokens: st.SearchStrategy[str] = st.builds( + lambda head, tail: head + tail, + st.sampled_from("abcdefghijklmnopqrstuvwxyz0123456789"), + st.text(alphabet="abcdefghijklmnopqrstuvwxyz0123456789+.-", max_size=11), +) +_apt_flag_tokens: st.SearchStrategy[str] = st.builds( + lambda word: "--" + word, + st.text(alphabet="abcdefghijklmnopqrstuvwxyz-", min_size=1, max_size=12), +) +_apt_install_tokens: st.SearchStrategy[list[tuple[str, bool]]] = st.lists( + st.one_of( + st.tuples(_apt_package_tokens, st.just(False)), + st.tuples(_apt_flag_tokens, st.just(True)), + ), + min_size=1, + max_size=6, +) + + +@given( + tagged=_apt_install_tokens, + use_sudo=st.booleans(), + indent=st.text(alphabet=" \t", max_size=3), +) +def test_extract_apt_install_packages_recovers_non_flag_arguments( + tagged: list[tuple[str, bool]], use_sudo: bool, indent: str +) -> None: + """Return exactly the non-flag install arguments, in order, dropping flags.""" + tokens = [token for token, _ in tagged] + expected = [token for token, is_flag in tagged if not is_flag] + prefix = "sudo " if use_sudo else "" + setup_commands = ( + "export DEBIAN_FRONTEND=noninteractive\n" + f"{indent}{prefix}apt-get install {' '.join(tokens)}\n" + ) + + assert _extract_apt_install_packages(setup_commands) == expected + + +@given( + packages=st.lists(_apt_package_tokens, min_size=1, max_size=5), + indent=st.text(alphabet=" \t", max_size=3), +) +def test_extract_apt_install_packages_skips_commented_installs( + packages: list[str], indent: str +) -> None: + """A commented apt-get install line never contributes packages.""" + setup_commands = ( + f"{indent}# sudo apt-get install --yes {' '.join(packages)}\necho noop\n" + ) + + assert _extract_apt_install_packages(setup_commands) == [] + + +@given(packages=st.lists(_apt_package_tokens, min_size=1, max_size=4)) +def test_extract_apt_install_packages_follows_line_continuations( + packages: list[str], +) -> None: + """Recover packages split onto a backslash-continued line.""" + setup_commands = f"sudo apt-get install --yes \\\n {' '.join(packages)}\n" + + assert _extract_apt_install_packages(setup_commands) == packages + + +def _mutation_workflow_with_schedule(schedule_block: str) -> str: + """Build a mutation-testing workflow whose ``on`` reaches the schedule check.""" + return f'permissions: {{}}\n"on":\n{schedule_block} workflow_dispatch:\n' + + +def test_assert_mutation_workflow_contracts_rejects_extra_schedule_entry() -> None: + """Reject a second schedule entry beyond the single sanctioned cron.""" + workflow = _mutation_workflow_with_schedule( + f' schedule:\n - cron: "{_MUTATION_CRON}"\n - cron: "0 0 * * *"\n' + ) + + with pytest.raises(AssertionError, match="schedule to be exactly one cron entry"): + _assert_mutation_workflow_contracts(workflow) + + +def test_assert_mutation_workflow_contracts_rejects_extra_schedule_key() -> None: + """Reject an extra key smuggled into the single schedule entry.""" + workflow = _mutation_workflow_with_schedule( + f' schedule:\n - cron: "{_MUTATION_CRON}"\n branches: ["main"]\n' + ) + + with pytest.raises(AssertionError, match="schedule to be exactly one cron entry"): + _assert_mutation_workflow_contracts(workflow) diff --git a/tests/test_template/test_tooling_contracts.py b/tests/test_template/test_tooling_contracts.py index 25383ed..7bb4066 100644 --- a/tests/test_template/test_tooling_contracts.py +++ b/tests/test_template/test_tooling_contracts.py @@ -59,6 +59,9 @@ def test_generated_tooling_contracts( coverage_main_workflow = read_generated_text( project / ".github/workflows/coverage-main.yml" ) + mutation_workflow = read_generated_text( + project / ".github/workflows/mutation-testing.yml" + ) docs_contents = read_generated_text(project / "docs/contents.md") repository_layout = read_generated_text(project / "docs/repository-layout.md") readme = read_generated_text(project / "README.md") @@ -89,6 +92,7 @@ def test_generated_tooling_contracts( ci_workflow=ci_workflow, audit_workflow=audit_workflow, act_workflow=act_workflow, + mutation_workflow=mutation_workflow, docs_contents=docs_contents, repository_layout=repository_layout, readme=readme,