From a7ab15cc6ef4c1cb63f97e5fe5e8064cea066a98 Mon Sep 17 00:00:00 2001 From: Simon Keimer Date: Sat, 19 Sep 2026 12:51:46 +0200 Subject: [PATCH] fix(trace): a gate verdict from ANY commit counted as run-status evidence (#1413) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `trace.py` now compares the stored verdict's `commit` against HEAD, degrading to the same "unverified" NOTE a toolchain mismatch already produces. Two probes added to `evidence-self-test`. `_gate_log`'s own comment states the rule — "A verdict is attributable to (tree, HEAD, TOOLCHAIN)" — and the code checked TWO of the three: INVALID (tree/HEAD moved DURING the run) and a toolchain mismatch. The commit was never compared. FOUND BY RELYING ON IT. #1412 added an enforced requirement with a fresh `// VERIFIES:` binding, and `scripts/trace.sh check` returned PASS on a verdict taken at an earlier commit — one that predated the binding. Sabotage-measured before the fix, with a control proving the machinery fires: control : verdict commit == HEAD -> TRACE: PASS rc=0 sabotage : verdict commit := deadbeef... -> TRACE: PASS rc=0 <-- unchecked control 2: verdict toolchain := bogus -> NOTE "...run a full gate" SCOPE, NARROWLY. Inside gate.sh this was never wrong: gate.sh:203 exports GATE_LOG, so the live log is used and is attributable by construction. In CI the traceability job has no target/ and already reports run-status unverified. The hole was the STANDALONE `trace.sh check` path — the one used while AUTHORING a binding, which is when the evidence claim is first made. Sharper failure it permitted: rename a test, bind the old name, and the stale log still records that name passing, so the binding validates against a test that no longer exists. DEGRADED, NOT FATAL, deliberately: the toolchain case returns (None, reason) and the caller reports a NOTE. Matching it gives "not attributable" one behaviour and keeps an ordinary edit-then-check cycle quiet. The probes are committed rather than performed once: evidence-self-test gains "PASS from another commit" and "PASS with no commit recorded", both required to be refused, alongside the existing PASS control that stops the check being satisfied by refusing everything. Verification-objective: a stored gate verdict may only vouch for a cited test's run-status when it is attributable to the tree being checked, so that a binding cannot be validated by a run that predates it. Review: none — mirroring the toolchain branch three lines above it, with the defect sabotage-measured first and the new behaviour probed in both directions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0188ATCj6DZ9aRVQ2vSirua6 --- docs/dev/project/traceability.md | 42 ++++++++++++++++++++++++++++++ scripts/lib/trace.py | 44 +++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/docs/dev/project/traceability.md b/docs/dev/project/traceability.md index 4323cfdd..b8362bfa 100644 --- a/docs/dev/project/traceability.md +++ b/docs/dev/project/traceability.md @@ -15,6 +15,48 @@ and the actually-observed results per change. --- +## 2026-09-19 — a gate verdict from ANY commit counted as run-status evidence; #1413 + +**Change.** `trace.py` compares the stored verdict's `commit` against HEAD, degrading to the same +"unverified" NOTE the toolchain mismatch already produces. Two probes added to `evidence-self-test`. + +**The defect, and how it was found.** `_gate_log`'s own comment states the rule — *"A verdict is +attributable to (tree, HEAD, TOOLCHAIN)"* — and the code checked **two** of the three: `INVALID` +(tree/HEAD moved *during* the run) and a toolchain mismatch. The commit was never compared. Found by +relying on it: #1412 added an `enforced` requirement with a fresh `// VERIFIES:` binding, and +`scripts/trace.sh check` returned PASS on a verdict taken at an earlier commit — one that predated +the binding. + +**Sabotage-measured before the fix, with a control proving the surrounding machinery fires:** + +``` +control : verdict commit == HEAD -> TRACE: PASS rc=0 +sabotage : verdict commit := deadbeef… -> TRACE: PASS rc=0 <-- unchecked +control 2: verdict toolchain := "rustc 0.0.0 (not real)" -> NOTE "…run a full gate" +``` + +**Scope, stated narrowly.** Inside `gate.sh` the check was never wrong: `gate.sh:203` exports +`GATE_LOG="$LOG"`, so the live log is used and is attributable by construction. In CI the +traceability job has no `target/`, so it already reports `no gate log in target/ — run-status of +enforced bindings unverified`. The hole was the **standalone** `scripts/trace.sh check` path — the +one a developer runs while *authoring* a binding, which is exactly when the evidence claim is first +made. The sharper failure it permitted: rename a test and bind the old name, and the stale log still +records that name passing, so the binding validates against a test that no longer exists. + +**Degraded, not fatal, deliberately.** The toolchain case returns `(None, reason)` and the caller +reports a NOTE; matching it gives "not attributable" a single behaviour and keeps an ordinary +edit-then-check cycle quiet. Verified both ways after the fix: a stale commit yields *"the last gate +verdict was taken at 0e42ea2f9d84 and HEAD is 7a3b3ea1fe64 — a verdict does not survive a commit +change; run a full gate"*, and a matching verdict yields zero NOTEs and PASS. + +**The probes are committed, not performed once.** `evidence-self-test` gains *"PASS from another +commit"* and *"PASS with no commit recorded"*, both required to be refused, alongside the existing +PASS control that stops the check being satisfied by refusing everything. The unrecorded case +mirrors the toolchain one: every verdict written before `gate.sh` recorded a commit looks like that +and must not be trusted by default. + +--- + ## 2026-09-19 — REQ-PTT-01's registered statement was a paraphrase that changed its meaning; REQ-PTT-04 added; #1411 **Change.** REQ-PTT-01's yaml statement restored to its ratified prose and re-pointed diff --git a/scripts/lib/trace.py b/scripts/lib/trace.py index 05955279..6fb825bb 100644 --- a/scripts/lib/trace.py +++ b/scripts/lib/trace.py @@ -366,6 +366,19 @@ def _log_is_complete(path): return saw_result and complete +def _current_commit(): + """`git rev-parse HEAD`, or None when it cannot be read. + + Returns None for the same reason `_current_toolchain` does: this EXPIRES evidence, and a tree + that cannot be interrogated must not have its stored verdict called stale on that account. + """ + try: + out = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.SubprocessError): + return None + return out.stdout.strip() if out.returncode == 0 else None + + def _current_toolchain(): """`rustc -V`, or None when it cannot be read. @@ -428,6 +441,21 @@ def _evidence_log(): was = got_tc or "an unrecorded toolchain" return None, (f"the last gate verdict was produced by {was} and this is {want} — a " f"verdict does not survive a toolchain change; run a full gate") + # THE THIRD MEMBER OF THE TRIPLE (#1413). The comment above names (tree, HEAD, TOOLCHAIN) + # and the code checked two of them: a verdict taken at ANY commit was accepted as proof that + # a cited test ran. Sabotage-measured before this existed — rewriting the stored commit to + # `deadbeef…` still gave TRACE: PASS. The failure it permitted: add a `// VERIFIES:` binding, + # skip the gate, and the checker confirms the test "ran and passed" from a run that predates + # the binding; sharper, rename a test and bind the old name, and the stale log still records + # that name passing. Degraded rather than fatal, exactly like the toolchain case, so that + # "not attributable" has one behaviour and an ordinary edit-then-check cycle is not noisy. + want_commit = _current_commit() + got_commit = v.get("commit") + if want_commit and got_commit != want_commit: + was = got_commit[:12] if got_commit else "an unrecorded commit" + return None, (f"the last gate verdict was taken at {was} and HEAD is " + f"{want_commit[:12]} — a verdict does not survive a commit change; " + f"run a full gate") named = v.get("log") if named and os.path.exists(named) and _log_is_complete(named): return named, None @@ -569,16 +597,24 @@ def run_with(text, env_log=True): else: try: here = _current_toolchain() + at = _current_commit() for label, fields, want_log in ( - ("INVALID", {"result": "INVALID", "toolchain": here}, False), - ("PASS", {"result": "PASS", "toolchain": here}, True), + ("INVALID", {"result": "INVALID", "toolchain": here, "commit": at}, False), + ("PASS", {"result": "PASS", "toolchain": here, "commit": at}, True), + # The (tree, HEAD, TOOLCHAIN) triple's SECOND member (#1413). Unchecked until then: + # a verdict from any commit vouched for a cited test, including one whose binding + # did not exist when that gate ran. The unrecorded case matches the toolchain one — + # every verdict written before gate.sh recorded a commit looks like this. + ("PASS from another commit", + {"result": "PASS", "toolchain": here, "commit": "dead" * 10}, False), + ("PASS with no commit recorded", {"result": "PASS", "toolchain": here}, False), # A verdict is attributable to (tree, HEAD, TOOLCHAIN). These two probe the third # member, which drifts with no act by anyone — a distro upgrade. The unrecorded # case is not hypothetical: every verdict written before gate.sh recorded the # toolchain looks exactly like this, and must not be trusted by default. ("PASS from another toolchain", - {"result": "PASS", "toolchain": "rustc 0.0.0 (not this host)"}, False), - ("PASS with no toolchain recorded", {"result": "PASS"}, False), + {"result": "PASS", "toolchain": "rustc 0.0.0 (not this host)", "commit": at}, False), + ("PASS with no toolchain recorded", {"result": "PASS", "commit": at}, False), ): fields["log"] = complete verdict.write_text(_json.dumps(fields), encoding="utf-8")