ci(coverage): the gate was measuring 6 of 33 tests — #72 only half-fixed it

.github/actions/rust-test/action.yml

@@ -121,7 +121,12 @@ runs:
         FEATURES="${{ inputs.features }}"
         PACKAGE="${{ inputs.package }}"
         PACKAGES="${{ inputs.packages }}"
-        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == *"lib-q-keccak"* || "$PACKAGES" == *"lib-q-keccak"* ]]; then
+        # Exact package match, NOT a substring. `*"lib-q-keccak"*` also matched the sibling
+        # crate lib-q-keccak-digest, which is a normal testable rlib (its tests run in
+        # .github/actions/test-sha3) -- so it silently took the no_std compile-check path and
+        # had its coverage skipped on every PR that touched it. $PACKAGES is a space-separated
+        # list whose entries may carry an @features suffix, so match on those boundaries.
+        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == "lib-q-keccak" || " $PACKAGES " == *" lib-q-keccak "* || " $PACKAGES " == *" lib-q-keccak@"* ]]; then
           echo "skip=true" >> $GITHUB_OUTPUT
         else
           echo "skip=false" >> $GITHUB_OUTPUT
@@ -157,13 +162,18 @@ runs:
         if [[ -n "$FEATURES" ]]; then
           FEAT_ARGS=(--features "$FEATURES")
         fi
-        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == *"lib-q-keccak"* || "$PACKAGES" == *"lib-q-keccak"* ]]; then
+        # Exact package match, NOT a substring. `*"lib-q-keccak"*` also matched the sibling
+        # crate lib-q-keccak-digest, which is a normal testable rlib (its tests run in
+        # .github/actions/test-sha3) -- so it silently took the no_std compile-check path and
+        # had its coverage skipped on every PR that touched it. $PACKAGES is a space-separated
+        # list whose entries may carry an @features suffix, so match on those boundaries.
+        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == "lib-q-keccak" || " $PACKAGES " == *" lib-q-keccak "* || " $PACKAGES " == *" lib-q-keccak@"* ]]; then
           echo "Running no_std or no_std-compatible compilation check"
           echo "Cannot run tests in no_std mode or for no_std-compatible packages"
           if [[ -n "$PACKAGE" && "$PACKAGE" != "" ]]; then
             echo "Testing specific package: $PACKAGE"
             # Check that the specific package compiles in no_std mode
-            if [[ "$PACKAGE" == *"lib-q-keccak"* ]]; then
+            if [[ "$PACKAGE" == "lib-q-keccak" ]]; then
               echo "Building no_std package $PACKAGE (rlib-only; no extra panic feature)"
               cargo check -p $PACKAGE --profile dev-no-std --no-default-features --features "${FEATURES:-alloc}" --verbose
             elif [[ "$PACKAGE" == "lib-q-core" || "$PACKAGE" == "lib-q-random" ]]; then
@@ -262,13 +272,18 @@ runs:
         if [[ -n "$FEATURES" ]]; then
           FEAT_ARGS=(--features "$FEATURES")
         fi
-        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == *"lib-q-keccak"* || "$PACKAGES" == *"lib-q-keccak"* ]]; then
+        # Exact package match, NOT a substring. `*"lib-q-keccak"*` also matched the sibling
+        # crate lib-q-keccak-digest, which is a normal testable rlib (its tests run in
+        # .github/actions/test-sha3) -- so it silently took the no_std compile-check path and
+        # had its coverage skipped on every PR that touched it. $PACKAGES is a space-separated
+        # list whose entries may carry an @features suffix, so match on those boundaries.
+        if [[ "$FEATURES" == *"no_std"* || "$PACKAGE" == "lib-q-keccak" || " $PACKAGES " == *" lib-q-keccak "* || " $PACKAGES " == *" lib-q-keccak@"* ]]; then
           echo "Running no_std or no_std-compatible release compilation check"
           echo "Cannot run tests in no_std mode or for no_std-compatible packages"
           if [[ -n "$PACKAGE" && "$PACKAGE" != "" ]]; then
             echo "Testing specific package in release: $PACKAGE"
             # Check that the specific package compiles in no_std mode for release
-            if [[ "$PACKAGE" == *"lib-q-keccak"* ]]; then
+            if [[ "$PACKAGE" == "lib-q-keccak" ]]; then
               echo "Building no_std package $PACKAGE in release mode (rlib-only)"
               cargo check -p $PACKAGE --profile release --no-default-features --features "${FEATURES:-alloc}" --verbose
             elif [[ "$PACKAGE" == "lib-q-core" || "$PACKAGE" == "lib-q-random" ]]; then

.github/workflows/ci.yml

@@ -59,6 +59,15 @@ jobs:
 
       - name: Guard banned terms in new primitive crates
         run: bash ./scripts/ci-guard-primitive-banned-terms.sh
+
+      # Static (it never runs tarpaulin) and deliberately in core-validation rather than a coverage
+      # job: it must run on EVERY PR, including the ones that never trigger a tarpaulin run. The
+      # gaps it catches (a test-name filter shrinking the numerator, a package silently skipped,
+      # source hidden from the denominator in a nested crate, the denominator narrowed head-on)
+      # are invisible in a green coverage report. ~13s wall clock measured on a Windows dev box;
+      # not measured on this Linux runner, but it is a repo walk plus one bash process.
+      - name: Guard coverage-gate scope (numerator, selection, denominator)
+        run: bash ./scripts/ci-guard-coverage-honesty.sh
 
       - name: Core validation
         uses: ./.github/actions/rust-build

docs/coverage-scope.md

@@ -7,13 +7,80 @@ This document defines how [test-coverage.md](test-coverage.md) policy maps to **
 | Tier | Intent | Policy target | Current CI gate (see workflows) |
 |------|--------|---------------|----------------------------------|
 | Core library slice | `lib-q-core` sources under `lib-q-core/src`, excluding `wasm/` in PR coverage | ≥80% line on cryptographic API, validation, providers | **78%** line: PR `test-coverage` when `lib-q-core` is the affected package, the `lib-q-core` step in [coverage.yml](../.github/workflows/coverage.yml), and [`effective_threshold_for`](../scripts/verify-workspace-coverage.sh) for local workspace sweeps. |
-| Other affected crates | The package chosen by PR `test-coverage` when the diff hits a listed prefix (see below), or the `lib-q` / `lib-q-core` fallback | ≥80% line (policy); gates ratchet toward that | PR `test-coverage`: default **70%** line; **exceptions** (still below 70% measured portable line): `lib-q-ml-dsa` **60%**, `lib-q-keccak`/`lib-q-kem` **65%**, `lib-q-zkp` **65%**, `lib-q-sig` **66%**, `lib-q-hpke` **66%**, `lib-q-aead`/`lib-q-cb-kem` **68%**. [coverage.yml](../.github/workflows/coverage.yml) (push to `main`, path-filtered PRs, weekly schedule) uses the same exception list plus default **70** for all other crates in its scripted batches (including `lib-q-fn-dsa`, which has no lowered floor yet). |
+| Other affected crates | The package chosen by PR `test-coverage` when the diff hits a listed prefix (see below), or the `lib-q` / `lib-q-core` fallback | ≥80% line (policy); gates ratchet toward that | PR `test-coverage`: default **70%** line; **exceptions** (still below 70% measured portable line): `lib-q-ml-dsa` **60%**, `lib-q-keccak`/`lib-q-kem` **65%**, `lib-q-zkp` **65%**, `lib-q-sig` **66%**, `lib-q-hpke` **66%**, `lib-q-aead`/`lib-q-cb-kem` **68%**. [coverage.yml](../.github/workflows/coverage.yml) (push to `main`, path-filtered PRs, weekly schedule) uses the same exception list plus default **70** for all other crates in its scripted batches, and additionally floors `lib-q-fn-dsa` at **68%**. |
 | Security-critical subset | `lib-q-sig/src/lib.rs`, `ml_dsa.rs`, `provider.rs` (same sources built for `std`+`ml-dsa`) | ≥95% line, 100% branch when tooling emits branches | [security-critical-coverage.yml](../.github/workflows/security-critical-coverage.yml): **70%** line on that scoped set (other `src/*.rs` files are feature-gated and are not part of the denominator); **100%** branch when reported |
 
 **PR package selection:** [.github/workflows/pr.yml](../.github/workflows/pr.yml) picks a single package by scanning the diff against ordered lists (`CORE_CRATES`, `CRYPTO_CRATES`, `UTIL_CRATES`, then Stark/plonk workspace members). The first matching prefix wins. If none match, the job tests `lib-q` when `lib-q/`, `Cargo.toml`, or `Cargo.lock` changed, and otherwise defaults to `lib-q-core`. Workspace members outside those lists are not individually targeted by this job.
 
 For any PR package other than the umbrella `lib-q`, tarpaulin scopes `--include-files` to that package’s own sources (conventionally `<crate>/src/**`, or `examples/*.rs` for the example-only `lib-q-examples` member) so Cobertura `line-rate` is not dominated by dependency code. Resolution is shared by [scripts/print-tarpaulin-include-args.sh](../scripts/print-tarpaulin-include-args.sh) (used from the `rust-test` action and [scripts/run-coverage.sh](../scripts/run-coverage.sh)); CI fails the coverage step if a non-empty `-p`/`--packages` target would run without `--include-files`. Exceptions: `lib-q-core` additionally excludes other member crates and `wasm/` under PR settings; `lib-q-keccak` also excludes `advanced_simd.rs` (nightly/simd-only) plus `x86_simd_avx512.rs` and `x86.rs` (the AVX-512 batched permutation and the x86 SIMD absorption entrypoints — `target_feature`-gated, so a runner without AVX-512/AVX2 takes the scalar fallback and never executes the intrinsic bodies); `lib-q-ml-dsa` excludes `src/simd/avx2.rs`, the `src/simd/avx2/` tree, and `src/ml_dsa_generic/instantiations/avx2.rs` because those sources are built only with `simd256`, while default coverage runs use the portable backend; `lib-q-intrinsics` excludes the opposite-ISA file for the runner (`arm64.rs` on x86_64, `avx2.rs` on aarch64, both on other architectures). AVX2/simd256 behavior is still covered by tests in [.github/workflows/ci.yml](../.github/workflows/ci.yml) (`ml-dsa-compliance`, e.g. `determinism` with `simd256`). The scheduled/push [Test Coverage workflow](../.github/workflows/coverage.yml) also runs a second, **non-gated** tarpaulin pass for `lib-q-ml-dsa` with `--ml-dsa-simd256` (stable only); reports land under `combined-coverage/.../crypto/lib-q-ml-dsa-simd256/`. Local equivalent: `bash scripts/run-coverage.sh --crate lib-q-ml-dsa --ml-dsa-simd256 --threshold 0 --output-dir coverage-ml-dsa-avx2`. To sweep every workspace package from `cargo metadata`: [scripts/verify-workspace-coverage.sh](../scripts/verify-workspace-coverage.sh) — `effective_threshold_for` mirrors the per-crate floors in [pr.yml](../.github/workflows/pr.yml) and [coverage.yml](../.github/workflows/coverage.yml) (including **78%** for `lib-q-core` and the lowered floors below **70** where applicable).
 
+## What the gate can silently stop measuring
+
+A coverage percentage is a fraction, and both halves can be corrupted without the number ever
+looking wrong. [scripts/ci-guard-coverage-honesty.sh](../scripts/ci-guard-coverage-honesty.sh)
+(run on every PR from `core-validation` in [ci.yml](../.github/workflows/ci.yml)) asserts the
+failure modes this repository has actually hit:
+
+1. **Numerator — test-name filters.** No `cargo tarpaulin` command may pass test *names* after
+   libtest's `--` separator; only scheduling/output flags (`--test-threads=1` for `lib-q-kem`) are
+   allowed. A name filter shrinks the set of tests that runs while `--include-files` leaves the
+   denominator untouched, so the result describes the filter. `lib-q-fn-dsa` carried
+   `-- keypair_generation test_basic_fn_dsa_functionality sign_and_verify seeded_sign`, which ran
+   6 of its 34 tests and measured **69/161 = 42.86%** against a 68% floor; with the filter removed
+   the same code measures **129/161 = 80.12%** (measured locally, `x86_64-pc-windows-msvc`,
+   cargo-tarpaulin 0.32.8 `--engine llvm`; CI's Linux figure will differ slightly).
+   The files to inspect are **discovered** by walking the repo for shell/YAML/PowerShell files that
+   mention tarpaulin, not read off a list, and command text is reached by tainting every variable
+   that flows into the invocation — so neither a new workflow nor a different append idiom
+   (`CMD+=`, or a filter parked in a variable named nothing like "cmd") escapes it.
+2. **Selection — silently skipped packages.** The `coverage-skip` step in the
+   [rust-test action](../.github/actions/rust-test/action.yml) matched `*"lib-q-keccak"*` as a
+   substring, which also swallowed the unrelated sibling `lib-q-keccak-digest`: it took the no_std
+   compile-check path and its coverage never ran on any PR. The predicate is now an exact match,
+   and the guard runs the **shipped** predicate against every workspace package **under every
+   input shape the action accepts** — `package:`, `packages:`, a `packages:` entry carrying an
+   `@features` suffix, a package inside a longer `packages:` list, and a non-`no_std` `features:`
+   string. Driving only `package:` would leave the `$PACKAGES` arm unexercised, and `ci.yml` really
+   does pass `packages:`. Only packages on an explicit allowlist may be skipped; `lib-q-keccak`
+   remains skipped there (no_std rlib under a panic=abort profile) and is gated by `coverage.yml`
+   at 65% instead.
+3. **Denominator — source hidden in nested crates.** `--include-files '<crate>/src/**'` cannot see
+   a nested cargo package. The guard fails on any nested package inside a workspace member that is
+   neither a member itself nor in `[workspace].exclude`, unless it is recorded as a known gap.
+4. **Denominator — narrowed head-on.** A whole-crate `--include-files` must name a directory glob
+   (`<crate>/src/*`, `<crate>/src/**`, `*.rs`); pointing it at `<crate>/src/lib.rs` shrinks the
+   denominator to one file. Deliberately scoped tiers are allowed from `NARROW_INCLUDE_ALLOWLIST`,
+   keyed by *file* so the security-critical tier's narrow includes cannot license the same
+   narrowing in the whole-crate gate. Symmetrically, an `--exclude-files` reaching inside a
+   member's own `src/` must appear in `SRC_EXCLUDE_ALLOWLIST` — the ~15 existing ones are all
+   code the runner cannot execute (SIMD/arch-gated bodies, non-compiled cfgs) and each carries its
+   reason there. Excluding a file that merely lacks tests now fails the build.
+
+**What the guard does not cover** (a green run is not a proof the number is right): it is static
+and never runs tarpaulin; CHECK 1's taint analysis is per-file, so a command assembled across two
+files is not modelled; CHECK 4 does not allowlist coarse `<crate>/*` exclusions such as the
+sibling-crate list `lib-q-core` uses, so an exclusion naming the crate under `--packages` would
+pass; and discovery keys on the literal string `tarpaulin`, so a wrapper that never spells the
+tool's name is invisible. These are recorded in the script header rather than papered over.
+
+### Known gap: FN-DSA nested crates
+
+`lib-q-fn-dsa`'s gated percentage describes `lib-q-fn-dsa/src/lib.rs` (161 measurable lines) and
+nothing else. The five nested crates `lib-q-fn-dsa/fn-dsa{,-comm,-kgen,-sign,-vrfy}` hold roughly
+37k lines and are outside the denominator — including `fn-dsa-kgen/src/poly.rs`, where a portable
+keygen livelock survived from 2026-05-17 to 2026-07-27 with no coverage signal. **Read
+"lib-q-fn-dsa: 80%" as a statement about the wrapper, not about FN-DSA.**
+
+They cannot simply be added to `--include-files`: they are not workspace members, so
+`tarpaulin --packages lib-q-fn-dsa` never runs their own test suites, and widening the denominator
+without running those suites would crater the figure and break the gate blind. Closing this is
+measure-then-gate: add report-only (`--threshold 0`) tarpaulin rows for each nested crate to
+`coverage.yml` — the precedent already exists there for `lib-q-ml-dsa --ml-dsa-simd256` — then set
+floors from what CI prints. Note that `--packages <name>` will not resolve for a non-member, so
+those rows need a manifest-path invocation or the crates need to become workspace members first;
+that has not been verified on a Linux runner. The entry in `NESTED_PACKAGE_EXCEPTIONS` keeps the
+gap visible until then.
+
 ## Security-critical paths (line targets)
 
 These are the first scoped paths used for the dedicated workflow; extend the list in that workflow when new stable entry points warrant it.

scripts/ci-guard-coverage-honesty.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Guard: the coverage gate must measure what it claims to measure.
+#
+# WHY THIS EXISTS
+# ---------------
+# A coverage percentage is a fraction. Both halves can be quietly corrupted, and when they are
+# the gate keeps going green/red for reasons that have nothing to do with the code:
+#
+#   * NUMERATOR   -- a test-NAME filter appended after libtest's `--` separator shrinks the set
+#                    of tests that runs, while --include-files (the denominator) stays put.
+#                    lib-q-fn-dsa carried `-- keypair_generation test_basic_fn_dsa_functionality
+#                    sign_and_verify seeded_sign`, which ran 6 of the crate's 34 tests and scored
+#                    69/161 = 42.86% against a 68% floor. Removing the filter: 129/161 = 80.12%.
+#                    The crate never changed. Only the filter did.
+#   * SELECTION   -- a package silently routed to the "skip coverage" branch is never measured at
+#                    all. `*"lib-q-keccak"*` is a substring pattern, so it also swallowed the
+#                    unrelated sibling crate lib-q-keccak-digest.
+#   * DENOMINATOR -- --include-files scoped to `<crate>/src/**` misses source that lives in a
+#                    NESTED cargo package under that crate. lib-q-fn-dsa/src/lib.rs is 834 lines;
+#                    lib-q-fn-dsa/fn-dsa-*/src is ~37k lines and is not in the denominator. The
+#                    two-month FN-DSA portable-keygen livelock lived in that excluded tree.
+#                    The same half can also be narrowed head-on, by pointing --include-files at a
+#                    single file or by adding an --exclude-files for source that merely lacks
+#                    tests. Both raise the percentage without a line of new test code.
+#
+# Each check below corresponds to one of those failure modes, and to a defect that was actually
+# found in this repository. Every check fails CLOSED: if it cannot locate what it is supposed to
+# inspect, it errors rather than silently passing.
+#
+# WHAT THIS GUARD DOES *NOT* COVER
+# --------------------------------
+# Stated so the next reader does not mistake a green run for a proof of correctness:
+#
+#   * It is STATIC. It reads the command lines CI would build; it never runs tarpaulin and cannot
+#     tell you that a percentage is right -- only that the fraction was not rescoped behind your
+#     back.
+#   * CHECK 1 reaches command text by tainting variables that flow into a `cargo tarpaulin`
+#     invocation *within one file*. A command assembled across two files (a helper that echoes a
+#     filter which the caller interpolates) is not modelled.
+#   * CHECK 4 only requires an allowlist entry for an --exclude-files that reaches inside a
+#     workspace member's own `src/`. A coarse `<crate>/*` exclusion -- the idiom lib-q-core uses to
+#     keep sibling-crate lines out of its Cobertura -- is NOT allowlisted, because 15 entries whose
+#     only failure mode is excluding the crate you are measuring is friction that buys little. If
+#     you ever see a `<crate>/*` exclusion naming the crate under `--packages`, that is the gap.
+#   * Discovery is by file suffix (.sh/.ps1/.yml/...) plus the literal string "tarpaulin". A
+#     tarpaulin command reached through a wrapper that never spells the tool's name is unseen.
+#
+# Usage: bash scripts/ci-guard-coverage-honesty.sh [REPO_ROOT]
+
+set -euo pipefail
+
+ROOT="${1:-$(git rev-parse --show-toplevel)}"
+cd "$ROOT"
+
+# Probe by RUNNING each candidate: on Windows a `python3` App Execution Alias sits on PATH and
+# satisfies `command -v` while refusing to execute.
+PY_BIN=""
+for candidate in python3 python py; do
+  if command -v "$candidate" >/dev/null 2>&1 && "$candidate" -c "import sys" >/dev/null 2>&1; then
+    PY_BIN="$candidate"
+    break
+  fi
+done
+if [[ -z "$PY_BIN" ]]; then
+  echo "ci-guard-coverage-honesty: a working python3 interpreter is required" >&2
+  exit 1
+fi
+
+"$PY_BIN" scripts/ci_guard_coverage_honesty.py "$ROOT"