From 5c1bf6196d92608fb70947f9db8d314468575cef Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Sun, 6 Sep 2026 23:50:05 +0900 Subject: [PATCH 1/9] fix(ci): require every release test to reach a gate --- .github/workflows/ci.yml | 3 +++ CONTRIBUTING.md | 6 +++++ scripts/check_test_reachability.sh | 39 ++++++++++++++++++++++++++++++ scripts/ci-local.sh | 1 + 4 files changed, 49 insertions(+) create mode 100755 scripts/check_test_reachability.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c531effc..80e85fdb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: - name: Check required repository files run: bash scripts/check_repo_completeness.sh + - name: Check every release and E2E test is invoked by a gate + run: bash scripts/check_test_reachability.sh + - name: Check release version consistency run: bash scripts/check_release_versions.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 097290cb..1a094dbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -155,6 +155,12 @@ deterministic source-relative check, while external URLs stay bounded to `scripts/markdown-link-exclusions.txt` only when a source file must be skipped; the discovery test rejects exclusions that no longer name a tracked file. +**Every release and E2E test must be reachable from a gate.** +`scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and +`tests/e2e/*.test.sh`, then requires each exact path to appear in `ci.yml`, +`e2e.yml`, `release.yml`, or `ci-local.sh`. Add the invocation in the same +change as a new test; an uninvoked test makes both local and remote CI fail. + **A change that touches no Rust skips the Rust gate.** The workspace suite exists to stop a Rust regression reaching `main`, and a diff with no `.rs` file in it cannot cause one: diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh new file mode 100755 index 00000000..4154d62c --- /dev/null +++ b/scripts/check_test_reachability.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +gate_files=( + "$repo_root/.github/workflows/ci.yml" + "$repo_root/.github/workflows/e2e.yml" + "$repo_root/.github/workflows/release.yml" + "$repo_root/scripts/ci-local.sh" +) + +check_suite() { + local suite="$1" + local test_file relative_path + local tests=() + + while IFS= read -r -d '' test_file; do + tests+=("$test_file") + done < <(find "$repo_root/tests/$suite" -maxdepth 1 -type f -name '*.test.sh' -print0) + + if (( ${#tests[@]} == 0 )); then + printf 'test-reachability: no tests discovered in tests/%s/*.test.sh\n' "$suite" >&2 + return 1 + fi + + for test_file in "${tests[@]}"; do + relative_path="${test_file#"$repo_root/"}" + if ! grep -Fq -- "$relative_path" "${gate_files[@]}"; then + printf 'test-reachability: test is not invoked by a gate: %s\n' \ + "$relative_path" >&2 + return 1 + fi + done +} + +check_suite release +check_suite e2e + +printf 'test-reachability: every release and E2E test is invoked by a gate.\n' diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 40771ee0..a21381ff 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -261,6 +261,7 @@ hygiene_shellcheck() ( run_hygiene_group() { printf '\n### hygiene\n' run_step 'hygiene: check_repo_completeness.sh' bash "$repo_root/scripts/check_repo_completeness.sh" + run_step 'hygiene: check_test_reachability.sh' bash "$repo_root/scripts/check_test_reachability.sh" run_step 'hygiene: check_release_versions.sh' bash "$repo_root/scripts/check_release_versions.sh" run_step 'hygiene: public-claims.test.sh' bash "$repo_root/tests/release/public-claims.test.sh" run_step 'hygiene: npm test --prefix packages/setup' npm test --prefix "$repo_root/packages/setup" From c0291a7ea19ecbac577ce09285d4402529df51a0 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Tue, 8 Sep 2026 08:27:22 +0900 Subject: [PATCH 2/9] test(ci): verify the reachability gate --- .github/workflows/ci.yml | 3 ++ scripts/check_test_reachability.sh | 2 ++ scripts/ci-local.sh | 1 + tests/release/test-reachability.test.sh | 41 +++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100755 tests/release/test-reachability.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80e85fdb..ca8b981e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Check every release and E2E test is invoked by a gate run: bash scripts/check_test_reachability.sh + - name: Test the reachability gate + run: bash tests/release/test-reachability.test.sh + - name: Check release version consistency run: bash scripts/check_release_versions.sh diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index 4154d62c..90299983 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -31,6 +31,8 @@ check_suite() { return 1 fi done + + return 0 } check_suite release diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index a21381ff..53cdf356 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -262,6 +262,7 @@ run_hygiene_group() { printf '\n### hygiene\n' run_step 'hygiene: check_repo_completeness.sh' bash "$repo_root/scripts/check_repo_completeness.sh" run_step 'hygiene: check_test_reachability.sh' bash "$repo_root/scripts/check_test_reachability.sh" + run_step 'hygiene: test-reachability.test.sh' bash "$repo_root/tests/release/test-reachability.test.sh" run_step 'hygiene: check_release_versions.sh' bash "$repo_root/scripts/check_release_versions.sh" run_step 'hygiene: public-claims.test.sh' bash "$repo_root/tests/release/public-claims.test.sh" run_step 'hygiene: npm test --prefix packages/setup' npm test --prefix "$repo_root/packages/setup" diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh new file mode 100755 index 00000000..68f1e7fe --- /dev/null +++ b/tests/release/test-reachability.test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fixture="$(mktemp -d)" +trap 'rm -rf "$fixture"' EXIT + +mkdir -p "$fixture/.github/workflows" "$fixture/scripts" \ + "$fixture/tests/release" "$fixture/tests/e2e" +cp "$repo_root/scripts/check_test_reachability.sh" "$fixture/scripts/" +touch "$fixture/tests/release/reachable.test.sh" "$fixture/tests/e2e/reachable.test.sh" +cat > "$fixture/.github/workflows/ci.yml" <<'EOF' +run: bash tests/release/reachable.test.sh +run: bash tests/e2e/reachable.test.sh +EOF +touch "$fixture/.github/workflows/e2e.yml" "$fixture/.github/workflows/release.yml" \ + "$fixture/scripts/ci-local.sh" + +bash "$fixture/scripts/check_test_reachability.sh" "$fixture" >/dev/null + +touch "$fixture/tests/release/orphan.test.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then + printf 'test-reachability test: orphan test unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'test is not invoked by a gate: tests/release/orphan.test.sh' <<< "$output" || { + printf 'test-reachability test: orphan diagnostic omitted its path: %s\n' "$output" >&2 + exit 1 +} + +rm "$fixture/tests/release/orphan.test.sh" "$fixture/tests/e2e/reachable.test.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then + printf 'test-reachability test: empty suite unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'no tests discovered in tests/e2e/*.test.sh' <<< "$output" || { + printf 'test-reachability test: empty-suite diagnostic missing: %s\n' "$output" >&2 + exit 1 +} + +printf 'test-reachability test: orphan and empty-suite failures validated.\n' From a95af9fbbb048da074d1da41747f4d5d82cb8c6d Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Wed, 9 Sep 2026 08:21:59 +0900 Subject: [PATCH 3/9] test: exclude local helper from release gates --- scripts/check_test_reachability.sh | 1 - tests/release/test-reachability.test.sh | 13 ++++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index 90299983..e3420a67 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -6,7 +6,6 @@ gate_files=( "$repo_root/.github/workflows/ci.yml" "$repo_root/.github/workflows/e2e.yml" "$repo_root/.github/workflows/release.yml" - "$repo_root/scripts/ci-local.sh" ) check_suite() { diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh index 68f1e7fe..4e56ed11 100755 --- a/tests/release/test-reachability.test.sh +++ b/tests/release/test-reachability.test.sh @@ -38,4 +38,15 @@ grep -Fq 'no tests discovered in tests/e2e/*.test.sh' <<< "$output" || { exit 1 } -printf 'test-reachability test: orphan and empty-suite failures validated.\n' +touch "$fixture/tests/e2e/reachable.test.sh" "$fixture/tests/release/local-only.test.sh" +printf '%s\n' 'run: bash tests/release/local-only.test.sh' >> "$fixture/scripts/ci-local.sh" +if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then + printf 'test-reachability test: local-only test unexpectedly passed\n' >&2 + exit 1 +fi +grep -Fq 'test is not invoked by a gate: tests/release/local-only.test.sh' <<< "$output" || { + printf 'test-reachability test: local-only diagnostic omitted its path: %s\n' "$output" >&2 + exit 1 +} + +printf 'test-reachability test: orphan, empty-suite, and local-only failures validated.\n' From 9f89c0af5a55cb61c55e22c3cf5f62d0680be3c6 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Fri, 11 Sep 2026 11:02:37 +0900 Subject: [PATCH 4/9] docs: align release gate guidance --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a094dbb..ee510c7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -158,7 +158,7 @@ the discovery test rejects exclusions that no longer name a tracked file. **Every release and E2E test must be reachable from a gate.** `scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and `tests/e2e/*.test.sh`, then requires each exact path to appear in `ci.yml`, -`e2e.yml`, `release.yml`, or `ci-local.sh`. Add the invocation in the same +`e2e.yml`, or `release.yml`. Add the invocation in the same change as a new test; an uninvoked test makes both local and remote CI fail. **A change that touches no Rust skips the Rust gate.** The workspace suite exists From 6894fdd46a84513eb109eda904f15bfe9cf3ca24 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Tue, 15 Sep 2026 03:47:02 +0900 Subject: [PATCH 5/9] fix(ci): match standalone release test invocations --- CONTRIBUTING.md | 9 +++-- scripts/check_test_reachability.sh | 17 ++++++++- tests/release/test-reachability.test.sh | 51 ++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f3e0941..7adeec0e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -181,9 +181,12 @@ the discovery test rejects exclusions that no longer name a tracked file. **Every release and E2E test must be reachable from a gate.** `scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and -`tests/e2e/*.test.sh`, then requires each exact path to appear in `ci.yml`, -`e2e.yml`, or `release.yml`. Add the invocation in the same -change as a new test; an uninvoked test makes both local and remote CI fail. +`tests/e2e/*.test.sh`, then requires a standalone inline `run: bash ` +step for each exact path in `ci.yml`, `e2e.yml`, or `release.yml`. Trailing +comments are allowed; comments alone, artifact paths, shell compound commands, +and mentions in `ci-local.sh` do not count. Keep these test steps in that +explicit form and add the invocation in the same change as a new test; a test +without one makes both local and remote CI fail. **A change that touches no Rust skips the Rust gate.** The workspace suite exists to stop a Rust regression reaching `main`, and a diff with no `.rs` file in it diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index e3420a67..56ce650b 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -repo_root="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" gate_files=( "$repo_root/.github/workflows/ci.yml" "$repo_root/.github/workflows/e2e.yml" @@ -24,7 +24,20 @@ check_suite() { for test_file in "${tests[@]}"; do relative_path="${test_file#"$repo_root/"}" - if ! grep -Fq -- "$relative_path" "${gate_files[@]}"; then + # Require the standalone inline command used by our workflows. Merely + # naming a test in a comment, artifact path, or another command is not + # an invocation. Compare the path literally, not as a regular expression. + if ! awk -v path="$relative_path" ' + { + line = $0 + if (sub(/^[[:space:]]*(-[[:space:]]+)?run:[[:space:]]+bash[[:space:]]+/, "", line)) { + sub(/[[:space:]]+#.*$/, "", line) + sub(/[[:space:]]+$/, "", line) + if (line == path) found = 1 + } + } + END { exit !found } + ' "${gate_files[@]}"; then printf 'test-reachability: test is not invoked by a gate: %s\n' \ "$relative_path" >&2 return 1 diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh index 4e56ed11..2dd4d99b 100755 --- a/tests/release/test-reachability.test.sh +++ b/tests/release/test-reachability.test.sh @@ -16,10 +16,10 @@ EOF touch "$fixture/.github/workflows/e2e.yml" "$fixture/.github/workflows/release.yml" \ "$fixture/scripts/ci-local.sh" -bash "$fixture/scripts/check_test_reachability.sh" "$fixture" >/dev/null +bash "$fixture/scripts/check_test_reachability.sh" >/dev/null touch "$fixture/tests/release/orphan.test.sh" -if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then printf 'test-reachability test: orphan test unexpectedly passed\n' >&2 exit 1 fi @@ -29,7 +29,7 @@ grep -Fq 'test is not invoked by a gate: tests/release/orphan.test.sh' <<< "$out } rm "$fixture/tests/release/orphan.test.sh" "$fixture/tests/e2e/reachable.test.sh" -if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then printf 'test-reachability test: empty suite unexpectedly passed\n' >&2 exit 1 fi @@ -40,7 +40,7 @@ grep -Fq 'no tests discovered in tests/e2e/*.test.sh' <<< "$output" || { touch "$fixture/tests/e2e/reachable.test.sh" "$fixture/tests/release/local-only.test.sh" printf '%s\n' 'run: bash tests/release/local-only.test.sh' >> "$fixture/scripts/ci-local.sh" -if output="$(bash "$fixture/scripts/check_test_reachability.sh" "$fixture" 2>&1)"; then +if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then printf 'test-reachability test: local-only test unexpectedly passed\n' >&2 exit 1 fi @@ -49,4 +49,45 @@ grep -Fq 'test is not invoked by a gate: tests/release/local-only.test.sh' <<< " exit 1 } -printf 'test-reachability test: orphan, empty-suite, and local-only failures validated.\n' +rm "$fixture/tests/release/local-only.test.sh" +touch "$fixture/tests/release/not-executed.test.sh" +cp "$fixture/.github/workflows/ci.yml" "$fixture/base-ci.yml" +for mention in \ + '# run: bash tests/release/not-executed.test.sh' \ + 'path: tests/release/not-executed.test.sh' \ + 'path: tests/release/not-executed.test.sh*' \ + 'run: echo bash tests/release/not-executed.test.sh' \ + 'run: bash tests/release/not-executed.test.sh.backup' \ + 'run: bash prefix/tests/release/not-executed.test.sh' \ + 'run: bash tests/release/not-executedXtestXsh' \ + 'run: bash tests/release/not-executed.test.sh || true'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf '%s\n' "$mention" >> "$fixture/.github/workflows/ci.yml" + if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then + printf 'test-reachability test: non-invocation unexpectedly passed: %s\n' "$mention" >&2 + exit 1 + fi + grep -Fq 'test is not invoked by a gate: tests/release/not-executed.test.sh' <<< "$output" +done + +# Each supported workflow can satisfy the gate with a standalone invocation. +for workflow in ci e2e release; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf '%s\n' ' - run: bash tests/release/not-executed.test.sh # run the test' \ + >> "$fixture/.github/workflows/$workflow.yml" + bash "$fixture/scripts/check_test_reachability.sh" >/dev/null + if [[ "$workflow" != ci ]]; then + : > "$fixture/.github/workflows/$workflow.yml" + fi +done + +# A missing workflow must fail even when an earlier file contains every match. +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +printf '%s\n' 'run: bash tests/release/not-executed.test.sh' >> "$fixture/.github/workflows/ci.yml" +rm "$fixture/.github/workflows/release.yml" +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: missing workflow unexpectedly passed\n' >&2 + exit 1 +fi + +printf 'test-reachability test: orphan, empty-suite, local-only, and invocation matching validated.\n' From a40ddc71d0c47f7e4a7e2d8597a4b3f86006f7d2 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Tue, 15 Sep 2026 03:53:21 +0900 Subject: [PATCH 6/9] fix(ci): inspect parsed workflow run commands --- CONTRIBUTING.md | 14 +++++--- scripts/check_test_reachability.sh | 45 +++++++++++++++++-------- tests/release/test-reachability.test.sh | 41 +++++++++++++++++++--- 3 files changed, 77 insertions(+), 23 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7adeec0e..ad64aa16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -181,12 +181,18 @@ the discovery test rejects exclusions that no longer name a tracked file. **Every release and E2E test must be reachable from a gate.** `scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and -`tests/e2e/*.test.sh`, then requires a standalone inline `run: bash ` +`tests/e2e/*.test.sh`, then requires a standalone `bash ` command in a `run` step for each exact path in `ci.yml`, `e2e.yml`, or `release.yml`. Trailing comments are allowed; comments alone, artifact paths, shell compound commands, -and mentions in `ci-local.sh` do not count. Keep these test steps in that -explicit form and add the invocation in the same change as a new test; a test -without one makes both local and remote CI fail. +heredoc text, and mentions in `ci-local.sh` do not count. Keep these test steps +in that explicit form and add the invocation in the same change as a new test; +a test without one makes both local and remote CI fail. This is a static +invocation check; it does not evaluate job conditions or prove runtime execution. + +The workflow parser uses PyYAML, already installed with CI's `yamllint` +prerequisite. Install it into the same Python environment used to run the gate: +`python3 -m pip install yamllint==1.38.0`. A missing parser or unreadable workflow +fails the gate instead of falling back to text matching. **A change that touches no Rust skips the Rust gate.** The workspace suite exists to stop a Rust regression reaching `main`, and a diff with no `.rs` file in it diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index 56ce650b..e5bdfd9a 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -8,6 +8,36 @@ gate_files=( "$repo_root/.github/workflows/release.yml" ) +# PyYAML is installed by the existing yamllint prerequisite. Parse actual run +# fields so strings in action inputs, comments, and heredocs cannot count. +invoked_tests="$(python3 - "${gate_files[@]}" <<'PYTHON' +import re +import sys + +try: + import yaml +except ImportError: + sys.exit("test-reachability: PyYAML is required; install yamllint with python3 -m pip install yamllint==1.38.0") + +command = re.compile(r"bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)[ \t]*(?:#.*)?") +for path in sys.argv[1:]: + try: + with open(path, encoding="utf-8") as stream: + workflow = yaml.safe_load(stream) + for job in (workflow or {}).get("jobs", {}).values(): + for step in job.get("steps", []): + if not isinstance(step, dict): + continue + run = step.get("run") + if isinstance(run, str): + match = command.fullmatch(run.strip()) + if match: + print(match.group(1)) + except (OSError, yaml.YAMLError, AttributeError, TypeError) as error: + sys.exit(f"test-reachability: cannot read workflow {path}: {error}") +PYTHON +)" + check_suite() { local suite="$1" local test_file relative_path @@ -24,20 +54,7 @@ check_suite() { for test_file in "${tests[@]}"; do relative_path="${test_file#"$repo_root/"}" - # Require the standalone inline command used by our workflows. Merely - # naming a test in a comment, artifact path, or another command is not - # an invocation. Compare the path literally, not as a regular expression. - if ! awk -v path="$relative_path" ' - { - line = $0 - if (sub(/^[[:space:]]*(-[[:space:]]+)?run:[[:space:]]+bash[[:space:]]+/, "", line)) { - sub(/[[:space:]]+#.*$/, "", line) - sub(/[[:space:]]+$/, "", line) - if (line == path) found = 1 - } - } - END { exit !found } - ' "${gate_files[@]}"; then + if ! grep -Fxq -- "$relative_path" <<< "$invoked_tests"; then printf 'test-reachability: test is not invoked by a gate: %s\n' \ "$relative_path" >&2 return 1 diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh index 2dd4d99b..46d1aa27 100755 --- a/tests/release/test-reachability.test.sh +++ b/tests/release/test-reachability.test.sh @@ -10,8 +10,12 @@ mkdir -p "$fixture/.github/workflows" "$fixture/scripts" \ cp "$repo_root/scripts/check_test_reachability.sh" "$fixture/scripts/" touch "$fixture/tests/release/reachable.test.sh" "$fixture/tests/e2e/reachable.test.sh" cat > "$fixture/.github/workflows/ci.yml" <<'EOF' -run: bash tests/release/reachable.test.sh -run: bash tests/e2e/reachable.test.sh +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: bash tests/release/reachable.test.sh + - run: bash tests/e2e/reachable.test.sh EOF touch "$fixture/.github/workflows/e2e.yml" "$fixture/.github/workflows/release.yml" \ "$fixture/scripts/ci-local.sh" @@ -62,7 +66,7 @@ for mention in \ 'run: bash tests/release/not-executedXtestXsh' \ 'run: bash tests/release/not-executed.test.sh || true'; do cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" - printf '%s\n' "$mention" >> "$fixture/.github/workflows/ci.yml" + printf ' - name: Not an invocation\n %s\n' "$mention" >> "$fixture/.github/workflows/ci.yml" if output="$(bash "$fixture/scripts/check_test_reachability.sh" 2>&1)"; then printf 'test-reachability test: non-invocation unexpectedly passed: %s\n' "$mention" >&2 exit 1 @@ -70,10 +74,37 @@ for mention in \ grep -Fq 'test is not invoked by a gate: tests/release/not-executed.test.sh' <<< "$output" done +# Scalar text can look like a step while only being data passed to an action. +for scalar in '|' '>' '"'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf '%s\n' ' - uses: actions/upload-artifact@v4' ' with:' \ + " path: $scalar" ' run: bash tests/release/not-executed.test.sh' \ + >> "$fixture/.github/workflows/ci.yml" + if [[ "$scalar" == '"' ]]; then + printf ' "\n' >> "$fixture/.github/workflows/ci.yml" + fi + if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: artifact scalar unexpectedly passed: %s\n' "$scalar" >&2 + exit 1 + fi +done +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + cat <<'TEXT' + run: bash tests/release/not-executed.test.sh + TEXT +EOF +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: heredoc text unexpectedly passed\n' >&2 + exit 1 +fi + # Each supported workflow can satisfy the gate with a standalone invocation. for workflow in ci e2e release; do cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" - printf '%s\n' ' - run: bash tests/release/not-executed.test.sh # run the test' \ + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/$workflow.yml" + printf '%s\n' ' - run: bash tests/release/not-executed.test.sh # run the test' \ >> "$fixture/.github/workflows/$workflow.yml" bash "$fixture/scripts/check_test_reachability.sh" >/dev/null if [[ "$workflow" != ci ]]; then @@ -83,7 +114,7 @@ done # A missing workflow must fail even when an earlier file contains every match. cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" -printf '%s\n' 'run: bash tests/release/not-executed.test.sh' >> "$fixture/.github/workflows/ci.yml" +printf '%s\n' ' - run: bash tests/release/not-executed.test.sh' >> "$fixture/.github/workflows/ci.yml" rm "$fixture/.github/workflows/release.yml" if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then printf 'test-reachability test: missing workflow unexpectedly passed\n' >&2 From 4a6e89798c10e574047451a45b3f08bc81caf5c4 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Tue, 15 Sep 2026 03:55:04 +0900 Subject: [PATCH 7/9] fix(ci): distinguish shell comments from filename suffixes --- scripts/check_test_reachability.sh | 2 +- tests/release/test-reachability.test.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index e5bdfd9a..f3a273e0 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -19,7 +19,7 @@ try: except ImportError: sys.exit("test-reachability: PyYAML is required; install yamllint with python3 -m pip install yamllint==1.38.0") -command = re.compile(r"bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)[ \t]*(?:#.*)?") +command = re.compile(r"bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)(?:[ \t]+#.*)?[ \t]*") for path in sys.argv[1:]: try: with open(path, encoding="utf-8") as stream: diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh index 46d1aa27..1154eeb5 100755 --- a/tests/release/test-reachability.test.sh +++ b/tests/release/test-reachability.test.sh @@ -62,6 +62,7 @@ for mention in \ 'path: tests/release/not-executed.test.sh*' \ 'run: echo bash tests/release/not-executed.test.sh' \ 'run: bash tests/release/not-executed.test.sh.backup' \ + 'run: bash tests/release/not-executed.test.sh#backup' \ 'run: bash prefix/tests/release/not-executed.test.sh' \ 'run: bash tests/release/not-executedXtestXsh' \ 'run: bash tests/release/not-executed.test.sh || true'; do From e6f73f003b31392e9df9abc648d4706ba7511121 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Wed, 16 Sep 2026 09:50:24 +0900 Subject: [PATCH 8/9] fix(ci): recognize non-interactive sudo test steps --- CONTRIBUTING.md | 6 ++++- scripts/check_test_reachability.sh | 2 +- tests/release/test-reachability.test.sh | 32 +++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad64aa16..2caf873f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -183,11 +183,15 @@ the discovery test rejects exclusions that no longer name a tracked file. `scripts/check_test_reachability.sh` discovers `tests/release/*.test.sh` and `tests/e2e/*.test.sh`, then requires a standalone `bash ` command in a `run` step for each exact path in `ci.yml`, `e2e.yml`, or `release.yml`. Trailing -comments are allowed; comments alone, artifact paths, shell compound commands, +comments and a `sudo` or `sudo -n` prefix are allowed. Other sudo options, +comments alone, artifact paths, shell compound commands, heredoc text, and mentions in `ci-local.sh` do not count. Keep these test steps in that explicit form and add the invocation in the same change as a new test; a test without one makes both local and remote CI fail. This is a static invocation check; it does not evaluate job conditions or prove runtime execution. +A YAML block scalar containing just that command is supported. Multi-line shell +scripts, including a command with a separate comment line, are not; give each +test its own standalone step instead. The workflow parser uses PyYAML, already installed with CI's `yamllint` prerequisite. Install it into the same Python environment used to run the gate: diff --git a/scripts/check_test_reachability.sh b/scripts/check_test_reachability.sh index f3a273e0..2eee52b8 100755 --- a/scripts/check_test_reachability.sh +++ b/scripts/check_test_reachability.sh @@ -19,7 +19,7 @@ try: except ImportError: sys.exit("test-reachability: PyYAML is required; install yamllint with python3 -m pip install yamllint==1.38.0") -command = re.compile(r"bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)(?:[ \t]+#.*)?[ \t]*") +command = re.compile(r"(?:sudo(?:[ \t]+-n)?[ \t]+)?bash[ \t]+(tests/(?:release|e2e)/[A-Za-z0-9_.-]+\.test\.sh)(?:[ \t]+#.*)?[ \t]*") for path in sys.argv[1:]: try: with open(path, encoding="utf-8") as stream: diff --git a/tests/release/test-reachability.test.sh b/tests/release/test-reachability.test.sh index 1154eeb5..3d57912d 100755 --- a/tests/release/test-reachability.test.sh +++ b/tests/release/test-reachability.test.sh @@ -61,6 +61,12 @@ for mention in \ 'path: tests/release/not-executed.test.sh' \ 'path: tests/release/not-executed.test.sh*' \ 'run: echo bash tests/release/not-executed.test.sh' \ + 'run: echo sudo bash tests/release/not-executed.test.sh' \ + 'run: sudo -l bash tests/release/not-executed.test.sh' \ + 'run: sudo -v bash tests/release/not-executed.test.sh' \ + 'run: sudo -nl bash tests/release/not-executed.test.sh' \ + 'run: sudo -n -l bash tests/release/not-executed.test.sh' \ + 'run: sudo -n bash tests/release/not-executed.test.sh#backup' \ 'run: bash tests/release/not-executed.test.sh.backup' \ 'run: bash tests/release/not-executed.test.sh#backup' \ 'run: bash prefix/tests/release/not-executed.test.sh' \ @@ -101,6 +107,32 @@ if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then exit 1 fi +# Privileged test steps must still be command invocations, not sudo queries. +for prefix in 'sudo' 'sudo -n'; do + cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" + printf ' - run: %s bash tests/release/not-executed.test.sh\n' "$prefix" \ + >> "$fixture/.github/workflows/ci.yml" + bash "$fixture/scripts/check_test_reachability.sh" >/dev/null +done + +# A block scalar with one command is supported, but a multi-line script is not. +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + sudo -n bash tests/release/not-executed.test.sh +EOF +bash "$fixture/scripts/check_test_reachability.sh" >/dev/null +cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" +cat >> "$fixture/.github/workflows/ci.yml" <<'EOF' + - run: | + # This remains a multi-line script even with only one command. + sudo -n bash tests/release/not-executed.test.sh +EOF +if bash "$fixture/scripts/check_test_reachability.sh" >/dev/null 2>&1; then + printf 'test-reachability test: multi-line script unexpectedly passed\n' >&2 + exit 1 +fi + # Each supported workflow can satisfy the gate with a standalone invocation. for workflow in ci e2e release; do cp "$fixture/base-ci.yml" "$fixture/.github/workflows/ci.yml" From 1cbd5dd9ec93145ba8d96233d36b9ba7c929b309 Mon Sep 17 00:00:00 2001 From: eunwoo song Date: Wed, 16 Sep 2026 10:20:32 +0900 Subject: [PATCH 9/9] fix(ci): avoid running discovered reachability test twice --- CONTRIBUTING.md | 2 ++ scripts/ci-local.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 611cf608..e6289673 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,8 @@ invocation check; it does not evaluate job conditions or prove runtime execution A YAML block scalar containing just that command is supported. Multi-line shell scripts, including a command with a separate comment line, are not; give each test its own standalone step instead. +The local hygiene runner discovers these test files automatically; do not also +add explicit local invocations, which would run a test twice. The workflow parser uses PyYAML, already installed with CI's `yamllint` prerequisite. Install it into the same Python environment used to run the gate: diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 32d81fe1..71e18f12 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -289,7 +289,6 @@ run_hygiene_group() { run_step 'hygiene: firewall backend reporter fixtures' python3 "$repo_root/tests/test_firewall_state.py" run_step 'hygiene: check_repo_completeness.sh' bash "$repo_root/scripts/check_repo_completeness.sh" run_step 'hygiene: check_test_reachability.sh' bash "$repo_root/scripts/check_test_reachability.sh" - run_step 'hygiene: test-reachability.test.sh' bash "$repo_root/tests/release/test-reachability.test.sh" run_step 'hygiene: check_release_versions.sh' bash "$repo_root/scripts/check_release_versions.sh" run_step 'hygiene: npm test --prefix packages/setup' npm test --prefix "$repo_root/packages/setup" run_shell_tests