diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5be425c8a1..44bf0b4de5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,8 @@ jobs: run: scripts/tests/check-protocol-provenance.sh - name: Check tests do not contact n0 infrastructure run: scripts/tests/check-hermetic-tests.sh + - name: Check the flaky sweep watches exactly the flaky tests + run: scripts/tests/check-flaky-sweep-scope.sh - name: Check exact relay TLS provider bundles run: scripts/tests/check-relay-tls-features.sh - name: Check boundary checker contract diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 181262ef673..5b75dedd0e4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -116,10 +116,38 @@ jobs: run: | cargo nextest list --workspace ${{ env.FEATURES }} --lib --bins --tests --run-ignored ignored-only + # The `-E` filterset below only applies to the flaky sweep + # (`--run-ignored all`), which runs every #[ignore]d test. nextest cannot + # read the ignore REASON, so without it the sweep also runs tests ignored + # for reasons that are not flakiness. Those fail every single run, which + # made the daily sweep permanently red -- and an alarm that is always on + # cannot report a NEW flake, the only thing it exists for. It was red on + # 2026-07-29, -30 and -31. + # + # Excluded, because they are not flaky: + # integration_mainline dials the public BitTorrent mainline DHT; + # deliberately manual. Excluded by its + # binary, `mainline`. + # downloader_get_many_smoke ignored "todo" -- unimplemented. + # + # The 11 patchbay tests ignored "not yet passing" are already excluded by + # `default-filter = 'not binary(patchbay)'` in .config/nextest.toml, which + # still applies alongside -E (verified -- -E composes with it rather than + # replacing it). + # + # This is emitted by a GitHub expression rather than a shell variable on + # purpose: this step runs under pwsh on the windows matrix entries, where + # bash parameter expansion (`${VAR:+...}`) is not expansion at all -- pwsh + # reads it as an undefined variable and silently substitutes nothing, so + # the filter would quietly not apply on Windows. Single quotes survive + # both shells. + # + # scripts/tests/check-flaky-sweep-scope.sh keeps this list in step with + # the #[ignore] reasons in the tree. - name: run tests run: | mkdir -p output - cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --profile ci --run-ignored ${{ inputs.flaky && 'all' || 'default' }} ${{ inputs.flaky && '--verbose' || '' }} --no-fail-fast --message-format ${{ inputs.flaky && 'libtest-json' || 'human' }} > output/${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json + cargo nextest run --workspace ${{ env.FEATURES }} --lib --bins --tests --profile ci --run-ignored ${{ inputs.flaky && 'all' || 'default' }} ${{ inputs.flaky && '-E ''not test(downloader_get_many_smoke) and not binary(mainline)''' || '' }} ${{ inputs.flaky && '--verbose' || '' }} --no-fail-fast --message-format ${{ inputs.flaky && 'libtest-json' || 'human' }} > output/${{ matrix.name }}_${{ matrix.features }}_${{ matrix.rust }}.json env: RUST_LOG: ${{ runner.debug && 'TRACE' || 'DEBUG'}} NEXTEST_EXPERIMENTAL_LIBTEST_JSON: 1 diff --git a/scripts/tests/check-flaky-sweep-scope.sh b/scripts/tests/check-flaky-sweep-scope.sh new file mode 100755 index 00000000000..9c33f91947f --- /dev/null +++ b/scripts/tests/check-flaky-sweep-scope.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# The daily flaky sweep must run exactly the tests ignored FOR FLAKINESS. +# +# flaky.yaml runs tests.yaml with `--run-ignored all`, which runs every +# #[ignore]d test. nextest cannot read the ignore reason, so without a filter +# the sweep also runs tests ignored because they are manual, unimplemented, or +# known-broken. Those fail on every run, which is how the sweep came to be +# permanently red (2026-07-29, -30, -31) -- and a red-by-construction sweep +# cannot report a NEW flake, which is the only thing it exists to do. +# +# tests.yaml therefore excludes the non-flaky ignores by name. That list has to +# stay in step with the tree, in BOTH directions: +# +# - a new #[ignore = "todo"]-style test that nobody excludes puts the sweep +# back to permanently red; +# - a test ignored "flaky" that ends up excluded silently stops being +# watched, which is worse than the sweep being red, because it looks fine. +# +# This asserts both. Classification is by ignore reason: a reason beginning +# "flaky" means sweep it, anything else means exclude it. +set -euo pipefail + +cd "$(dirname "$0")/../.." + +workflow=".github/workflows/tests.yaml" +nextest_config=".config/nextest.toml" + +# The filterset is emitted by a GitHub expression on the nextest command line +# (not a shell variable -- this step runs under pwsh on Windows, where bash +# parameter expansion silently yields nothing). Grab that line. +filter_line=$(grep -E "cargo nextest run .*-E ''not " "$workflow" || true) +if [[ -z "$filter_line" ]]; then + echo "FAIL: no flaky-sweep -E filterset found on the nextest command in $workflow" >&2 + echo " Without it the sweep runs every ignored test and is red forever." >&2 + echo " Expected a \${{ inputs.flaky && '-E ''not ...''' || '' }} fragment." >&2 + exit 1 +fi + +# A shell-expansion form would parse here but silently do nothing on Windows. +# Comment lines are stripped first: the comment above that step documents this +# exact hazard, and matching the documentation would be a false positive. +if grep -vE '^[[:space:]]*#' "$workflow" | grep -qE '\$\{[A-Z_]+:[+-]'; then + echo "FAIL: $workflow uses bash parameter expansion (\${VAR:+...})." >&2 + echo " This step runs under pwsh on the windows matrix entries, which does" >&2 + echo " not expand it -- the value silently vanishes there. Emit the argument" >&2 + echo " from a \${{ }} GitHub expression instead." >&2 + exit 1 +fi + +failed=0 + +# Collect `#[ignore = "reason"]` plus the fn it applies to. Attributes may sit +# between the ignore and the fn (#[traced_test], #[cfg(...)]), so scan forward. +# vendor/ is excluded: those are third-party trees this sweep does not own. +scan=$(git ls-files '*.rs' | grep -v '^vendor/' | while read -r file; do + awk -v F="$file" ' + match($0, /#\[ignore[[:space:]]*=[[:space:]]*"/) { + line = $0 + sub(/.*#\[ignore[[:space:]]*=[[:space:]]*"/, "", line) + sub(/".*/, "", line) + reason = line + for (i = 1; i <= 8; i++) { + if ((getline nxt) <= 0) break + if (match(nxt, /fn [a-zA-Z0-9_]+/)) { + name = substr(nxt, RSTART + 3, RLENGTH - 3) + print F "\t" name "\t" reason + break + } + } + } + ' "$file" +done) + +if [[ -z "$scan" ]]; then + echo "FAIL: found no #[ignore = \"...\"] tests at all -- the scan is broken, not the tree." >&2 + exit 1 +fi + +while IFS=$'\t' read -r file name reason; do + [[ -z "$name" ]] && continue + + # patchbay tests are excluded wholesale by default-filter in nextest.toml. + if [[ "$file" == *patchbay* ]]; then + if ! grep -q "not binary(patchbay)" "$nextest_config"; then + echo "FAIL: $name ($file) relies on 'not binary(patchbay)' in $nextest_config, which is gone" >&2 + failed=1 + fi + continue + fi + + if [[ "$reason" == flaky* ]]; then + # Must NOT be excluded -- the sweep exists to watch these. + if grep -qF "$name" <<<"$filter_line"; then + echo "FAIL: $name is ignored \"$reason\" but is excluded from the flaky sweep" >&2 + echo " $file" >&2 + echo " A flaky test that the sweep does not run is unwatched, and looks fine." >&2 + failed=1 + fi + else + # Must be excluded, by test name or by its binary. + binary=$(basename "$file" .rs) + if ! grep -qF "$name" <<<"$filter_line" && ! grep -qF "binary($binary)" <<<"$filter_line"; then + echo "FAIL: $name is ignored \"$reason\" -- not flakiness -- but the sweep runs it" >&2 + echo " $file" >&2 + echo " It will fail every night and keep the sweep red, hiding real flakes." >&2 + echo " Add 'not test($name)' (or 'not binary($binary)') to FLAKY_EXCLUDE in $workflow." >&2 + failed=1 + fi + fi +done <<<"$scan" + +[[ $failed -ne 0 ]] && exit 1 + +swept=$(awk -F'\t' '$3 ~ /^flaky/ && $1 !~ /patchbay/' <<<"$scan" | wc -l) +excluded=$(awk -F'\t' '$3 !~ /^flaky/ && $1 !~ /patchbay/' <<<"$scan" | wc -l) +echo "ok: flaky sweep watches $swept flaky test(s); $excluded non-flaky ignore(s) excluded"