diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23dce4d4..fb8f572b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,12 @@ jobs: - name: Check required repository files run: bash scripts/check_repo_completeness.sh + - name: Check every release and E2E test is invoked by a gate + run: bash scripts/check_test_reachability.sh + + - name: Test the reachability gate + run: bash tests/release/test-reachability.test.sh + - name: Check release version consistency run: bash scripts/check_release_versions.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f27e2a7..e6289673 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -179,6 +179,27 @@ deterministic source-relative check, while external URLs stay bounded to `scripts/markdown-link-exclusions.txt` only when a source file must be skipped; the discovery test rejects exclusions that no longer name a tracked file. +**Every release and E2E test must be reachable from a gate.** +`scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and +`tests/e2e/*.test.sh`, then requires a standalone `bash ` command in a `run` +step for each exact path in `ci.yml`, `e2e.yml`, or `release.yml`. Trailing +comments and a `sudo` or `sudo -n` prefix are allowed. Other sudo options, +comments alone, artifact paths, shell compound commands, +heredoc text, and mentions in `ci-local.sh` do not count. Keep these test steps +in that explicit form and add the invocation in the same change as a new test; +a test without one makes both local and remote CI fail. This is a static +invocation check; it does not evaluate job conditions or prove runtime execution. +A YAML block scalar containing just that command is supported. Multi-line shell +scripts, including a command with a separate comment line, are not; give each +test its own standalone step instead. +The local hygiene runner discovers these test files automatically; do not also +add explicit local invocations, which would run a test twice. + +The workflow parser uses PyYAML, already installed with CI's `yamllint` +prerequisite. Install it into the same Python environment used to run the gate: +`python3 -m pip install yamllint==1.38.0`. A missing parser or unreadable workflow +fails the gate instead of falling back to text matching. + **A change that touches no Rust skips the Rust gate.** The workspace suite exists to stop a Rust regression reaching `main`, and a diff with no `.rs` file in it cannot cause one: diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh new file mode 100755 index 00000000..2eee52b8 --- /dev/null +++ b/scripts/check_test_reachability.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +gate_files=( + "$repo_root/.github/workflows/ci.yml" + "$repo_root/.github/workflows/e2e.yml" + "$repo_root/.github/workflows/release.yml" +) + +# PyYAML is installed by the existing yamllint prerequisite. Parse actual run +# fields so strings in action inputs, comments, and heredocs cannot count. +invoked_tests="$(python3 - "${gate_files[@]}" <<'PYTHON' +import re +import sys + +try: + import yaml +except ImportError: + sys.exit("test-reachability: PyYAML is required; install yamllint with python3 -m pip install yamllint==1.38.0") + +command = re.compile(r"(?:sudo(?:[ \t]+-n)?[ \t]+)?bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)(?:[ \t]+#.*)?[ \t]*") +for path in sys.argv[1:]: + try: + with open(path, encoding="utf-8") as stream: + workflow = yaml.safe_load(stream) + for job in (workflow or {}).get("jobs", {}).values(): + for step in job.get("steps", []): + if not isinstance(step, dict): + continue + run = step.get("run") + if isinstance(run, str): + match = command.fullmatch(run.strip()) + if match: + print(match.group(1)) + except (OSError, yaml.YAMLError, AttributeError, TypeError) as error: + sys.exit(f"test-reachability: cannot read workflow {path}: {error}") +PYTHON +)" + +check_suite() { + local suite="$1" + local test_file relative_path + local tests=() + + while IFS= read -r -d '' test_file; do + tests+=("$test_file") + done < <(find "$repo_root/tests/$suite" -maxdepth 1 -type f -name '*.test.sh' -print0) + + if (( ${#tests[@]} == 0 )); then + printf 'test-reachability: no tests discovered in tests/%s/*.test.sh\n' "$suite" >&2 + return 1 + fi + + for test_file in "${tests[@]}"; do + relative_path="${test_file#"$repo_root/"}" + if ! grep -Fxq -- "$relative_path" <<< "$invoked_tests"; then + printf 'test-reachability: test is not invoked by a gate: %s\n' \ + "$relative_path" >&2 + return 1 + fi + done + + return 0 +} + +check_suite release +check_suite e2e + +printf 'test-reachability: every release and E2E test is invoked by a gate.\n' diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 0ec09919..71e18f12 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -288,6 +288,7 @@ run_hygiene_group() { printf '\n### hygiene\n' run_step 'hygiene: firewall backend reporter fixtures' python3 "$repo_root/tests/test_firewall_state.py" run_step 'hygiene: check_repo_completeness.sh' bash "$repo_root/scripts/check_repo_completeness.sh" + run_step 'hygiene: check_test_reachability.sh' bash "$repo_root/scripts/check_test_reachability.sh" run_step 'hygiene: check_release_versions.sh' bash "$repo_root/scripts/check_release_versions.sh" run_step 'hygiene: npm test --prefix packages/setup' npm test --prefix "$repo_root/packages/setup" run_shell_tests diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh new file mode 100755 index 00000000..3d57912d --- /dev/null +++ b/tests/release/test-reachability.test.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fixture="$(mktemp -d)" +trap 'rm -rf "$fixture"' EXIT + +mkdir -p "$fixture/.github/workflows" "$fixture/scripts" \ + "$fixture/tests/release" "$fixture/tests/e2e" +cp "$repo_root/scripts/check_test_reachability.sh" "$fixture/scripts/" +touch "$fixture/tests/release/reachable.test.sh" "$fixture/tests/e2e/reachable.test.sh" +cat > "$fixture/.github/workflows/ci.yml" <<'EOF' +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: bash tests/release/reachable.test.sh + - run: bash tests/e2e/reachable.test.sh +EOF +touch "$fixture/.github/workflows/e2e.yml" "$fixture/.github/workflows/release.yml" \ + "$fixture/scripts/ci-local.sh" + +bash "$fixture/scripts/check_test_reachability.sh" >/dev/null + +touch "$fixture/tests/release/orphan.test.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then + printf 'test-reachability test: orphan test unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'test is not invoked by a gate: tests/release/orphan.test.sh' <<< "$output" || { + printf 'test-reachability test: orphan diagnostic omitted its path: %s\n' "$output" >&2 + exit 1 +} + +rm "$fixture/tests/release/orphan.test.sh" "$fixture/tests/e2e/reachable.test.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then + printf 'test-reachability test: empty suite unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'no tests discovered in tests/e2e/*.test.sh' <<< "$output" || { + printf 'test-reachability test: empty-suite diagnostic missing: %s\n' "$output" >&2 + exit 1 +} + +touch "$fixture/tests/e2e/reachable.test.sh" "$fixture/tests/release/local-only.test.sh" +printf '%s\n' 'run: bash tests/release/local-only.test.sh' >> "$fixture/scripts/ci-local.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then + printf 'test-reachability test: local-only test unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'test is not invoked by a gate: tests/release/local-only.test.sh' <<< "$output" || { + printf 'test-reachability test: local-only diagnostic omitted its path: %s\n' "$output" >&2 + exit 1 +} + +rm "$fixture/tests/release/local-only.test.sh" +touch "$fixture/tests/release/not-executed.test.sh" +cp "$fixture/.github/workflows/ci.yml" "$fixture/base-ci.yml" +for mention in \ + '# run: bash tests/release/not-executed.test.sh' \ + 'path: tests/release/not-executed.test.sh' \ + 'path: tests/release/not-executed.test.sh*' \ + 'run: echo bash tests/release/not-executed.test.sh' \ + 'run: echo sudo bash tests/release/not-executed.test.sh' \ + 'run: sudo -l bash tests/release/not-executed.test.sh' \ + 'run: sudo -v bash tests/release/not-executed.test.sh' \ + 'run: sudo -nl bash tests/release/not-executed.test.sh' \ + 'run: sudo -n -l bash tests/release/not-executed.test.sh' \ + 'run: sudo -n bash tests/release/not-executed.test.sh#backup' \ + 'run: bash tests/release/not-executed.test.sh.backup' \ + 'run: bash tests/release/not-executed.test.sh#backup' \ + 'run: bash prefix/tests/release/not-executed.test.sh' \ + 'run: bash tests/release/not-executedXtestXsh' \ + 'run: bash tests/release/not-executed.test.sh || true'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf ' - name: Not an invocation\n %s\n' "$mention" >> "$fixture/.github/workflows/ci.yml" + if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then + printf 'test-reachability test: non-invocation unexpectedly passed: %s\n' "$mention" >&2 + exit 1 + fi + grep -Fq 'test is not invoked by a gate: tests/release/not-executed.test.sh' <<< "$output" +done + +# Scalar text can look like a step while only being data passed to an action. +for scalar in '|' '>' '"'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf '%s\n' ' - uses: actions/upload-artifact@v4' ' with:' \ + " path: $scalar" ' run: bash tests/release/not-executed.test.sh' \ + >> "$fixture/.github/workflows/ci.yml" + if [[ "$scalar" == '"' ]]; then + printf ' "\n' >> "$fixture/.github/workflows/ci.yml" + fi + if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: artifact scalar unexpectedly passed: %s\n' "$scalar" >&2 + exit 1 + fi +done +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + cat <<'TEXT' + run: bash tests/release/not-executed.test.sh + TEXT +EOF +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: heredoc text unexpectedly passed\n' >&2 + exit 1 +fi + +# Privileged test steps must still be command invocations, not sudo queries. +for prefix in 'sudo' 'sudo -n'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf ' - run: %s bash tests/release/not-executed.test.sh\n' "$prefix" \ + >> "$fixture/.github/workflows/ci.yml" + bash "$fixture/scripts/check_test_reachability.sh" >/dev/null +done + +# A block scalar with one command is supported, but a multi-line script is not. +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + sudo -n bash tests/release/not-executed.test.sh +EOF +bash "$fixture/scripts/check_test_reachability.sh" >/dev/null +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + # This remains a multi-line script even with only one command. + sudo -n bash tests/release/not-executed.test.sh +EOF +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: multi-line script unexpectedly passed\n' >&2 + exit 1 +fi + +# Each supported workflow can satisfy the gate with a standalone invocation. +for workflow in ci e2e release; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/$workflow.yml" + printf '%s\n' ' - run: bash tests/release/not-executed.test.sh # run the test' \ + >> "$fixture/.github/workflows/$workflow.yml" + bash "$fixture/scripts/check_test_reachability.sh" >/dev/null + if [[ "$workflow" != ci ]]; then + : > "$fixture/.github/workflows/$workflow.yml" + fi +done + +# A missing workflow must fail even when an earlier file contains every match. +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +printf '%s\n' ' - run: bash tests/release/not-executed.test.sh' >> "$fixture/.github/workflows/ci.yml" +rm "$fixture/.github/workflows/release.yml" +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: missing workflow unexpectedly passed\n' >&2 + exit 1 +fi + +printf 'test-reachability test: orphan, empty-suite, local-only, and invocation matching validated.\n'