diff --git a/.coverage b/.coverage index 84c8fe98..2618a946 100644 Binary files a/.coverage and b/.coverage differ diff --git a/docs/mutation-mutmut-workflow.md b/docs/mutation-mutmut-workflow.md index 2a7869cd..60e1ef91 100644 --- a/docs/mutation-mutmut-workflow.md +++ b/docs/mutation-mutmut-workflow.md @@ -29,8 +29,12 @@ are not supported by this workflow's first release. `window-hours` and translate changed `*.py` files into mutant-name globs (mutmut 3.x rejects file paths as run arguments): with the default `module-prefix-strip` of `src/`, `src/pkg/mod.py` scopes the - run to `pkg.mod.*`. When nothing relevant changed, the run writes a - skip message and finishes in seconds. + run to `pkg.mod.*`. Test modules (`conftest.py`, `test_*`, `*_test`, + and files under a `test/` or `tests/` directory) are not mutable + source, so they are excluded from glob translation; a change window + containing only test files short-circuits as a no-op with a + "no mutable source" summary note. When nothing relevant changed, the + run writes a skip message and finishes in seconds. - **`workflow_dispatch` runs** bypass the guard and mutate everything in `source_paths`. There is no shard fan-out — mutmut has no shard equivalent — so size `timeout-minutes` to the suite. diff --git a/workflow_scripts/mutation_run_mutmut.py b/workflow_scripts/mutation_run_mutmut.py index 2be1a158..8e543ef1 100644 --- a/workflow_scripts/mutation_run_mutmut.py +++ b/workflow_scripts/mutation_run_mutmut.py @@ -18,7 +18,13 @@ step naturally. - Positional run arguments are mutant-name globs in module-path form (file paths are rejected), so changed files are translated to module - globs: ``src/pkg/mod.py`` becomes ``pkg.mod.*``. + globs: ``src/pkg/mod.py`` becomes ``pkg.mod.*``. Test modules are not + mutable source, so they are skipped; a scope of test files alone + short-circuits to a graceful no-op rather than aborting mutmut with an + empty filter. +- The reusable workflow's ``INPUT_*`` variables configure this script but + are stripped from the environment before ``mutmut run``, so they cannot + leak into the mutated project's in-process pytest baseline. - Results live in per-source ``.meta`` files under ``mutants/``; the parseable interface is ``mutmut results --all true``, which prints ``name: status`` lines. @@ -56,6 +62,7 @@ import collections import dataclasses import os +import re import shlex import sys import typing as typ @@ -64,6 +71,9 @@ from cyclopts import App, Parameter from plumbum import RETCODE, local +if typ.TYPE_CHECKING: + import collections.abc as cabc + if __package__: from .output import emit, fail else: @@ -71,6 +81,13 @@ app = App() +#: Prefix of the workflow-input environment variables. mutmut runs the +#: caller's test suite in-process as its baseline, inheriting this +#: process's environment; leaving ``INPUT_*`` set would let workflow +#: inputs (for example ``INPUT_MODULE_PREFIX_STRIP``) leak into the +#: mutated project's tests, so they are stripped before ``mutmut run``. +WORKFLOW_INPUT_ENV_PREFIX = "INPUT_" + #: Statuses that indicate the suite failed to kill a runnable mutant. SURVIVOR_STATUSES: frozenset[str] = frozenset({"survived", "no tests"}) @@ -92,15 +109,38 @@ class MutantResult: status: str +#: Stems marking a pytest test module: ``conftest``, ``test_*``, ``*_test``. +TEST_STEM_RE = re.compile(r"^(?:conftest|test_.*|.*_test)$") + +#: Directory names that hold test modules. +TEST_DIR_NAMES: frozenset[str] = frozenset({"test", "tests"}) + + +def _is_test_file(path: PurePosixPath) -> bool: + """Return True when ``path`` is a pytest test module, not source. + + mutmut only generates mutants for source files, so a change window of + test modules alone maps to no mutants and would abort ``mutmut run`` + with an empty filter. Files whose stem matches ``TEST_STEM_RE`` or + that sit under a ``test``/``tests`` directory are treated as + non-mutable and skipped. + """ + if TEST_STEM_RE.match(path.stem): + return True + return any(part in TEST_DIR_NAMES for part in path.parts[:-1]) + + def _module_glob_for(name: str, prefix_strip: str) -> str | None: """Translate one changed file into a mutant-name glob, or None. - ``__init__.py`` maps to its package; non-Python paths and paths that - reduce to nothing after stripping yield None. + ``__init__.py`` maps to its package; non-Python paths, test modules, + and paths that reduce to nothing after stripping yield None. """ path = PurePosixPath(name) if path.suffix != ".py": return None + if _is_test_file(path): + return None stripped_base = PurePosixPath(prefix_strip.rstrip("/")) if prefix_strip else None if stripped_base is not None and path.is_relative_to(stripped_base): path = path.relative_to(stripped_base) @@ -238,6 +278,16 @@ def _mutmut_command(version: str) -> list[str]: return ["run", "--with", f"mutmut=={version}", "mutmut"] +def _workflow_input_env_names(names: cabc.Iterable[str]) -> list[str]: + """Return the workflow-input variable names among ``names``. + + These are the ``INPUT_*`` variables the reusable workflow sets on the + run step; they configure this script but must not reach the mutated + project's test environment. + """ + return [name for name in names if name.startswith(WORKFLOW_INPUT_ENV_PREFIX)] + + def _run_mutation_testing( globs: list[str], mutmut_version: str, extra_args: str ) -> None: @@ -254,7 +304,15 @@ def _run_mutation_testing( *globs, ] emit("mutation_mutmut_command", ["uv", *run_arguments]) - code = local["uv"][run_arguments] & RETCODE(FG=True) + # mutmut runs the caller's test suite in-process as its baseline, + # inheriting the subprocess environment; strip the workflow's INPUT_* + # variables so they cannot leak into the mutated project's tests + # (shared-actions#369). The `with` block restores the environment on + # exit, leaving the later `mutmut results` call unaffected. + with local.env(): + for name in _workflow_input_env_names(list(local.env.keys())): + del local.env[name] + code = local["uv"][run_arguments] & RETCODE(FG=True) emit("mutation_mutmut_exit_code", code) if code != 0: emit( @@ -265,6 +323,23 @@ def _run_mutation_testing( raise SystemExit(code) +NO_SOURCE_SUMMARY = """## Mutation testing results (mutmut) + +Skipped: changed files contain no mutable source. +""" + + +def _write_no_source_summary(summary_path: str) -> None: + """Note a graceful no-op in the job summary when nothing is mutable. + + Reached when the changed-file scope contains only non-Python or test + files, so no mutant-name globs are produced; skipping keeps ``mutmut + run`` from aborting on an empty filter (agent-helper-scripts#74). + """ + with Path(summary_path).open("a", encoding="utf-8") as handle: + handle.write(NO_SOURCE_SUMMARY) + + def _publish_results(mutmut_version: str, results_file: str, summary_path: str) -> None: """Capture ``mutmut results``, then write the artefact and summary.""" results_text = local["uv"][ @@ -320,8 +395,9 @@ def main( globs = files_to_module_globs(files, module_prefix_strip) if files.strip() and not globs: - emit("mutation_mutmut_outcome", "no python files in scope") + emit("mutation_mutmut_outcome", "changed files contain no mutable source") Path(results_file).write_text("", encoding="utf-8") + _write_no_source_summary(summary_env) return _run_mutation_testing(globs, mutmut_version, extra_args) diff --git a/workflow_scripts/tests/test_mutation_run_mutmut.py b/workflow_scripts/tests/test_mutation_run_mutmut.py index 08b0a1ff..554f4f80 100644 --- a/workflow_scripts/tests/test_mutation_run_mutmut.py +++ b/workflow_scripts/tests/test_mutation_run_mutmut.py @@ -57,6 +57,30 @@ def test_non_python_and_bare_prefix_are_ignored(self) -> None: "non-Python paths and bare prefixes should produce no globs" ) + @pytest.mark.parametrize( + "path", + [ + "hooks/test_post_turn_quality_stop_hook.py", + "src/mypkg/calc_test.py", + "src/mypkg/conftest.py", + "src/mypkg/tests/helpers.py", + ], + ) + def test_test_files_are_excluded(self, path: str) -> None: + """Pytest test modules map to no glob: mutmut never mutates them.""" + assert run_mutmut.files_to_module_globs(path, "src/") == [], ( + "test modules are not mutable source and must not become globs" + ) + + def test_test_only_scope_drops_out_leaving_source(self) -> None: + """A source file survives while its sibling test file is dropped.""" + globs = run_mutmut.files_to_module_globs( + "src/mypkg/calc.py src/mypkg/test_calc.py", "src/" + ) + assert globs == ["mypkg.calc.*"], ( + "only the mutable source file should reach mutmut as a glob" + ) + class TestParseResults: """Parsing of ``mutmut results --all true`` output.""" @@ -116,13 +140,14 @@ def test_empty_results_render_message(self) -> None: def fake_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: """Install a fake ``uv`` on PATH that scripts run/results calls. - ``mutmut run`` invocations exit with ``FAKE_MUTMUT_RUN_EXIT``; - ``mutmut results`` invocations print canned results. All arguments - are appended to ``uv-args.txt``. + ``mutmut run`` invocations exit with ``FAKE_MUTMUT_RUN_EXIT`` and dump + their environment to ``uv-env.txt``; ``mutmut results`` invocations + print canned results. All arguments are appended to ``uv-args.txt``. """ bin_dir = tmp_path / "bin" bin_dir.mkdir() args_file = tmp_path / "uv-args.txt" + env_file = tmp_path / "uv-env.txt" results_text = RESULTS_TEXT.replace("\n", "\\n") script = bin_dir / "uv" script.write_text( @@ -131,7 +156,7 @@ def fake_uv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: f"printf '\\n' >> \"{args_file}\"\n" 'case "$*" in\n' f" *results*) printf '{results_text}' ;;\n" - ' *run*mutmut*) exit "${FAKE_MUTMUT_RUN_EXIT:-0}" ;;\n' + f' *run*mutmut*) env > "{env_file}"; exit "${{FAKE_MUTMUT_RUN_EXIT:-0}}" ;;\n' "esac\n", encoding="utf-8", ) @@ -147,7 +172,17 @@ class TestMainEntry: def _prepare( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> tuple[Path, Path]: - """Point summary and results outputs at temp files.""" + """Point summary and results outputs at temp files. + + Ambient ``INPUT_*`` variables are cleared first so a value the + caller sets in the real CI environment (for example + ``INPUT_MODULE_PREFIX_STRIP=""`` leaking into mutmut's in-process + pytest baseline) cannot override the documented script defaults + that each test means to exercise. + """ + for name in ("INPUT_FILES", "INPUT_MUTMUT_VERSION", "INPUT_EXTRA_ARGS"): + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv("INPUT_MODULE_PREFIX_STRIP", raising=False) summary_file = tmp_path / "summary.md" summary_file.touch() results_file = tmp_path / "results.txt" @@ -188,6 +223,51 @@ def test_failing_baseline_propagates_exit_code( "a failing mutmut run should propagate its exit code" ) + @POSIX_SHIMS_ONLY + def test_workflow_input_env_does_not_leak_into_mutmut( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_uv: Path + ) -> None: + """``INPUT_*`` vars are stripped before mutmut runs the baseline. + + Reproduces shared-actions#369: the caller sets + ``INPUT_MODULE_PREFIX_STRIP=""``; without sanitisation that value + reaches mutmut's in-process pytest baseline and breaks tests that + read the ambient environment. + """ + self._prepare(tmp_path, monkeypatch) + monkeypatch.setenv("INPUT_FILES", "src/mypkg/calc.py") + monkeypatch.setitem(local.env, "INPUT_FILES", "src/mypkg/calc.py") + monkeypatch.setenv("INPUT_MODULE_PREFIX_STRIP", "") + monkeypatch.setitem(local.env, "INPUT_MODULE_PREFIX_STRIP", "") + run_mutmut.app([]) + env_dump = (tmp_path / "uv-env.txt").read_text(encoding="utf-8") + leaked = [line for line in env_dump.splitlines() if line.startswith("INPUT_")] + assert not leaked, ( + f"no INPUT_* variable should reach the mutmut subprocess, found {leaked}" + ) + + def test_test_only_scope_short_circuits_with_summary( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_uv: Path + ) -> None: + """A test-only change window skips mutmut and notes the no-op. + + Reproduces agent-helper-scripts#74: a lone changed test file maps + to no mutants, so the run short-circuits as a graceful no-op + instead of aborting mutmut with an empty filter. + """ + summary_file, results_file = self._prepare(tmp_path, monkeypatch) + monkeypatch.setenv("INPUT_FILES", "hooks/test_post_turn_quality_stop_hook.py") + run_mutmut.app([]) + assert not fake_uv.exists(), ( + "uv should never be invoked when the scope has no mutable source" + ) + assert results_file.read_text(encoding="utf-8") == "", ( + "the results file should stay empty on the short-circuit path" + ) + assert "no mutable source" in summary_file.read_text(encoding="utf-8"), ( + "the job summary should record the graceful no-op" + ) + def test_scope_without_python_files_short_circuits( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_uv: Path ) -> None: