fix(ci): require every release test to reach a gate

.github/workflows/ci.yml

@@ -47,6 +47,12 @@ jobs:
       - name: Check required repository files
         run: bash scripts/check_repo_completeness.sh
 
+      - name: Check every release and E2E test is invoked by a gate
+        run: bash scripts/check_test_reachability.sh
+
+      - name: Test the reachability gate
+        run: bash tests/release/test-reachability.test.sh
+
       - name: Check release version consistency
         run: bash scripts/check_release_versions.sh
 

CONTRIBUTING.md

@@ -179,6 +179,27 @@ deterministic source-relative check, while external URLs stay bounded to
 `scripts/markdown-link-exclusions.txt` only when a source file must be skipped;
 the discovery test rejects exclusions that no longer name a tracked file.
 
+**Every release and E2E test must be reachable from a gate.**
+`scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and
+`tests/e2e/*.test.sh`, then requires a standalone `bash <path>` command in a `run`
+step for each exact path in `ci.yml`, `e2e.yml`, or `release.yml`. Trailing
+comments and a `sudo` or `sudo -n` prefix are allowed. Other sudo options,
+comments alone, artifact paths, shell compound commands,
+heredoc text, and mentions in `ci-local.sh` do not count. Keep these test steps
+in that explicit form and add the invocation in the same change as a new test;
+a test without one makes both local and remote CI fail. This is a static
+invocation check; it does not evaluate job conditions or prove runtime execution.
+A YAML block scalar containing just that command is supported. Multi-line shell
+scripts, including a command with a separate comment line, are not; give each
+test its own standalone step instead.
+The local hygiene runner discovers these test files automatically; do not also
+add explicit local invocations, which would run a test twice.
+
+The workflow parser uses PyYAML, already installed with CI's `yamllint`
+prerequisite. Install it into the same Python environment used to run the gate:
+`python3 -m pip install yamllint==1.38.0`. A missing parser or unreadable workflow
+fails the gate instead of falling back to text matching.
+
 **A change that touches no Rust skips the Rust gate.** The workspace suite exists
 to stop a Rust regression reaching `main`, and a diff with no `.rs` file in it
 cannot cause one:

scripts/check_test_reachability.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+gate_files=(
+    "$repo_root/.github/workflows/ci.yml"
+    "$repo_root/.github/workflows/e2e.yml"
+    "$repo_root/.github/workflows/release.yml"
+)
+
+# PyYAML is installed by the existing yamllint prerequisite. Parse actual run
+# fields so strings in action inputs, comments, and heredocs cannot count.
+invoked_tests="$(python3 - "${gate_files[@]}" <<'PYTHON'
+import re
+import sys
+
+try:
+    import yaml
+except ImportError:
+    sys.exit("test-reachability: PyYAML is required; install yamllint with python3 -m pip install yamllint==1.38.0")
+
+command = re.compile(r"(?:sudo(?:[ \t]+-n)?[ \t]+)?bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)(?:[ \t]+#.*)?[ \t]*")
+for path in sys.argv[1:]:
+    try:
+        with open(path, encoding="utf-8") as stream:
+            workflow = yaml.safe_load(stream)
+        for job in (workflow or {}).get("jobs", {}).values():
+            for step in job.get("steps", []):
+                if not isinstance(step, dict):
+                    continue
+                run = step.get("run")
+                if isinstance(run, str):
+                    match = command.fullmatch(run.strip())
+                    if match:
+                        print(match.group(1))
+    except (OSError, yaml.YAMLError, AttributeError, TypeError) as error:
+        sys.exit(f"test-reachability: cannot read workflow {path}: {error}")
+PYTHON
+)"
+
+check_suite() {
+    local suite="$1"
+    local test_file relative_path
+    local tests=()
+
+    while IFS= read -r -d '' test_file; do
+        tests+=("$test_file")
+    done < <(find "$repo_root/tests/$suite" -maxdepth 1 -type f -name '*.test.sh' -print0)
+
+    if (( ${#tests[@]} == 0 )); then
+        printf 'test-reachability: no tests discovered in tests/%s/*.test.sh\n' "$suite" >&2
+        return 1
+    fi
+
+    for test_file in "${tests[@]}"; do
+        relative_path="${test_file#"$repo_root/"}"
+        if ! grep -Fxq -- "$relative_path" <<< "$invoked_tests"; then
+            printf 'test-reachability: test is not invoked by a gate: %s\n' \
+                "$relative_path" >&2
+            return 1
+        fi
+    done
+
+    return 0
+}
+
+check_suite release
+check_suite e2e
+
+printf 'test-reachability: every release and E2E test is invoked by a gate.\n'

scripts/ci-local.sh

@@ -288,6 +288,7 @@ run_hygiene_group() {
     printf '\n### hygiene\n'
     run_step 'hygiene: firewall backend reporter fixtures' python3 "$repo_root/tests/test_firewall_state.py"
     run_step 'hygiene: check_repo_completeness.sh' bash "$repo_root/scripts/check_repo_completeness.sh"
+    run_step 'hygiene: check_test_reachability.sh' bash "$repo_root/scripts/check_test_reachability.sh"
     run_step 'hygiene: check_release_versions.sh' bash "$repo_root/scripts/check_release_versions.sh"
     run_step 'hygiene: npm test --prefix packages/setup' npm test --prefix "$repo_root/packages/setup"
     run_shell_tests

tests/release/test-reachability.test.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+fixture="$(mktemp -d)"
+trap 'rm -rf "$fixture"' EXIT
+
+mkdir -p "$fixture/.github/workflows" "$fixture/scripts" \
+    "$fixture/tests/release" "$fixture/tests/e2e"
+cp "$repo_root/scripts/check_test_reachability.sh" "$fixture/scripts/"
+touch "$fixture/tests/release/reachable.test.sh" "$fixture/tests/e2e/reachable.test.sh"
+cat > "$fixture/.github/workflows/ci.yml" <<'EOF'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - run: bash tests/release/reachable.test.sh
+      - run: bash tests/e2e/reachable.test.sh
+EOF
+touch "$fixture/.github/workflows/e2e.yml" "$fixture/.github/workflows/release.yml" \
+    "$fixture/scripts/ci-local.sh"
+
+bash "$fixture/scripts/check_test_reachability.sh" >/dev/null
+
+touch "$fixture/tests/release/orphan.test.sh"
+if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then
+    printf 'test-reachability test: orphan test unexpectedly passed\n' >&2
+    exit 1
+fi
+grep -Fq 'test is not invoked by a gate: tests/release/orphan.test.sh' <<< "$output" || {
+    printf 'test-reachability test: orphan diagnostic omitted its path: %s\n' "$output" >&2
+    exit 1
+}
+
+rm "$fixture/tests/release/orphan.test.sh" "$fixture/tests/e2e/reachable.test.sh"
+if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then
+    printf 'test-reachability test: empty suite unexpectedly passed\n' >&2
+    exit 1
+fi
+grep -Fq 'no tests discovered in tests/e2e/*.test.sh' <<< "$output" || {
+    printf 'test-reachability test: empty-suite diagnostic missing: %s\n' "$output" >&2
+    exit 1
+}
+
+touch "$fixture/tests/e2e/reachable.test.sh" "$fixture/tests/release/local-only.test.sh"
+printf '%s\n' 'run: bash tests/release/local-only.test.sh' >> "$fixture/scripts/ci-local.sh"
+if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then
+    printf 'test-reachability test: local-only test unexpectedly passed\n' >&2
+    exit 1
+fi
+grep -Fq 'test is not invoked by a gate: tests/release/local-only.test.sh' <<< "$output" || {
+    printf 'test-reachability test: local-only diagnostic omitted its path: %s\n' "$output" >&2
+    exit 1
+}
+
+rm "$fixture/tests/release/local-only.test.sh"
+touch "$fixture/tests/release/not-executed.test.sh"
+cp "$fixture/.github/workflows/ci.yml" "$fixture/base-ci.yml"
+for mention in \
+    '# run: bash tests/release/not-executed.test.sh' \
+    'path: tests/release/not-executed.test.sh' \
+    'path: tests/release/not-executed.test.sh*' \
+    'run: echo bash tests/release/not-executed.test.sh' \
+    'run: echo sudo bash tests/release/not-executed.test.sh' \
+    'run: sudo -l bash tests/release/not-executed.test.sh' \
+    'run: sudo -v bash tests/release/not-executed.test.sh' \
+    'run: sudo -nl bash tests/release/not-executed.test.sh' \
+    'run: sudo -n -l bash tests/release/not-executed.test.sh' \
+    'run: sudo -n bash tests/release/not-executed.test.sh#backup' \
+    'run: bash tests/release/not-executed.test.sh.backup' \
+    'run: bash tests/release/not-executed.test.sh#backup' \
+    'run: bash prefix/tests/release/not-executed.test.sh' \
+    'run: bash tests/release/not-executedXtestXsh' \
+    'run: bash tests/release/not-executed.test.sh || true'; do
+    cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+    printf '      - name: Not an invocation\n        %s\n' "$mention" >> "$fixture/.github/workflows/ci.yml"
+    if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then
+        printf 'test-reachability test: non-invocation unexpectedly passed: %s\n' "$mention" >&2
+        exit 1
+    fi
+    grep -Fq 'test is not invoked by a gate: tests/release/not-executed.test.sh' <<< "$output"
+done
+
+# Scalar text can look like a step while only being data passed to an action.
+for scalar in '|' '>' '"'; do
+    cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+    printf '%s\n' '      - uses: actions/upload-artifact@v4' '        with:' \
+        "          path: $scalar" '            run: bash tests/release/not-executed.test.sh' \
+        >> "$fixture/.github/workflows/ci.yml"
+    if [[ "$scalar" == '"' ]]; then
+        printf '            "\n' >> "$fixture/.github/workflows/ci.yml"
+    fi
+    if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then
+        printf 'test-reachability test: artifact scalar unexpectedly passed: %s\n' "$scalar" >&2
+        exit 1
+    fi
+done
+cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+cat >> "$fixture/.github/workflows/ci.yml" <<'EOF'
+      - run: |
+          cat <<'TEXT'
+          run: bash tests/release/not-executed.test.sh
+          TEXT
+EOF
+if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then
+    printf 'test-reachability test: heredoc text unexpectedly passed\n' >&2
+    exit 1
+fi
+
+# Privileged test steps must still be command invocations, not sudo queries.
+for prefix in 'sudo' 'sudo -n'; do
+    cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+    printf '      - run: %s bash tests/release/not-executed.test.sh\n' "$prefix" \
+        >> "$fixture/.github/workflows/ci.yml"
+    bash "$fixture/scripts/check_test_reachability.sh" >/dev/null
+done
+
+# A block scalar with one command is supported, but a multi-line script is not.
+cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+cat >> "$fixture/.github/workflows/ci.yml" <<'EOF'
+      - run: |
+          sudo -n bash tests/release/not-executed.test.sh
+EOF
+bash "$fixture/scripts/check_test_reachability.sh" >/dev/null
+cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+cat >> "$fixture/.github/workflows/ci.yml" <<'EOF'
+      - run: |
+          # This remains a multi-line script even with only one command.
+          sudo -n bash tests/release/not-executed.test.sh
+EOF
+if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then
+    printf 'test-reachability test: multi-line script unexpectedly passed\n' >&2
+    exit 1
+fi
+
+# Each supported workflow can satisfy the gate with a standalone invocation.
+for workflow in ci e2e release; do
+    cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+    cp "$fixture/base-ci.yml" "$fixture/.github/workflows/$workflow.yml"
+    printf '%s\n' '      - run: bash tests/release/not-executed.test.sh  # run the test' \
+        >> "$fixture/.github/workflows/$workflow.yml"
+    bash "$fixture/scripts/check_test_reachability.sh" >/dev/null
+    if [[ "$workflow" != ci ]]; then
+        : > "$fixture/.github/workflows/$workflow.yml"
+    fi
+done
+
+# A missing workflow must fail even when an earlier file contains every match.
+cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml"
+printf '%s\n' '      - run: bash tests/release/not-executed.test.sh' >> "$fixture/.github/workflows/ci.yml"
+rm "$fixture/.github/workflows/release.yml"
+if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then
+    printf 'test-reachability test: missing workflow unexpectedly passed\n' >&2
+    exit 1
+fi
+
+printf 'test-reachability test: orphan, empty-suite, local-only, and invocation matching validated.\n'