From d5ded06c33669757916dfc770706d14bdafa9a0f Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Mon, 7 Sep 2026 20:15:35 +0000 Subject: [PATCH 01/13] fix(admission): narrow a mint's scan to the bundle that publishes the predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `admission_anchor` re-runs the rule a refusal named so it can recover that finding's fingerprint. `--rule` carries a PREDICATE id, so filtering on `declared.id == rule` selected nothing and the mint silently bound the head (CLOUD-1087, CLOUD-1125). Widening an empty exact match to every `policy` row fixed that and cost 2m22.121s per mint, measured on main at 6eb08e14 against 0.123s for a full adjudication of the same tree. "Widens to those rows and no further" bounds which KIND of row, and that is not the same as which ROW. `protected-mutation` is an engine-side rule name with zero hits under policy/, so no bundle could ever publish it — yet 58 modules were evaluated over the whole tree to build findings the next line discards. One test case paying it was 24% of the entire suite. `policy::publishers_of` asks each bundle its own published set, which is the same authority `attribute` resolves a violation's id against and the one `lint.rs` already reads for waiver-names-no-rule (CLOUD-1553). Where nothing publishes the predicate there is nothing to scan for, so the scan is skipped entirely rather than run and thrown away. Silent, deliberately: reporting the rule as unknown would decide CLOUD-1551's open question by accident. Measured here: loading and compiling every bundle is 2.4s against the 2m22s full run, so the expense was always tree acquisition and evaluation — which is exactly what this skips. The load is also the same work `run_over` does internally, so no module is compiled that would not have been. Not a revert. CLOUD-1087/CLOUD-1125's property is preserved by construction: a predicate a bundle DOES publish still selects that bundle's row and still anchors on its finding. The narrowing is extracted as a pure function rather than written inline because `admission_anchor` reads stdin and is reachable only through `override request`. Its tier's last case is not a fixture: it loads the REAL committed bundles and refuses any module publishing `protected-mutation`, with the filed-here/filed-over-own-diff pair asserted first so a repository whose bundles failed to load cannot pass it vacuously. Refs: CLOUD-1571 --- crates/batten/src/lib.rs | 70 +++++- crates/batten/src/policy.rs | 30 +++ crates/batten/tests/it/admission_narrowing.rs | 228 ++++++++++++++++++ crates/batten/tests/it/main.rs | 1 + 4 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 crates/batten/tests/it/admission_narrowing.rs diff --git a/crates/batten/src/lib.rs b/crates/batten/src/lib.rs index 8c7c58d21..4d4264a7c 100644 --- a/crates/batten/src/lib.rs +++ b/crates/batten/src/lib.rs @@ -5626,12 +5626,80 @@ fn admission_anchor( // exact match widens to those rows and no further. Every typed kind keeps // the narrow fast path it was given, which is what the ~90s measurement // above is about. + // + // AND "TO THOSE ROWS" IS A STATEMENT ABOUT WHICH *KIND*, WHICH IS NOT THE + // SAME AS WHICH ROW — the difference cost 2m22s per mint (CLOUD-1571). + // + // Widening to every `policy` row ran every policy module over the whole + // tree, whether or not any of them *could* publish the predicate being + // minted against. Measured on `main` at `6eb08e14`: `override request + // --rule protected-mutation` took **2m22.121s**, against **0.123s** for a + // full adjudication of the same tree, and one test case paying it was 24% + // of the entire suite. `protected-mutation` is an engine-side rule name + // with zero hits under `policy/`, so no bundle could ever have published + // it: 58 modules were evaluated to produce findings the filter below + // discards one line later. + // + // `Bundle::declared` is the module's own published predicate set, so + // asking it is what turns "which kind" into "which row". It is the same + // authority `attribute` resolves a violation's id against and the one + // `lint.rs` reads for `waiver-names-no-rule` (CLOUD-1553), on identical + // reasoning: reading the published set adds no second authority over what a + // bundle declares, where re-deriving it from module source would. + // + // THE LOAD IS NOT THE COST, and that is why this is affordable rather than + // merely narrower. Compiling every bundle is **2.4s** measured here against + // the 2m22s above — the expense is acquiring the tree and evaluating over + // it, which is exactly what this now skips. The load is also the same work + // `run_over` does internally, so no module is compiled that would not have + // been. + // + // CLOUD-1087/CLOUD-1125's property is untouched: a predicate a bundle DOES + // publish still selects that bundle's row and still anchors on its finding. + // This narrows the widening; it does not restore the `declared.id == rule` + // filter that made every policy admission a silent no-op. let selected: Vec<_> = if exact.is_empty() { - config + let policy_rows: Vec<_> = config .rules .iter() .filter(|declared| declared.kind == rules::RuleKind::Policy) .cloned() + .collect(); + // COULD-NOT-LOOK FALLS BACK rather than refusing, which is the posture + // the `run_over` call below already takes for its own `Err`: a bundle + // set that will not load leaves no honest way to ask who publishes the + // predicate, and `head()` is never weaker than what shipped. + let Ok(bundles) = policy::load( + root, + &policy_rows, + policy::Vocabulary { + patterns: &config.patterns, + verdicts: &config.verdicts, + recorders: &config.recorders, + }, + // The same entitlement the run below is given. A mint answers about + // one rule over one subject, so registry equality's exhausted half — + // a property of the whole authority — is not this verb's to assert. + policy::ModuleChecks::RunOverSelection, + None, + ) else { + return head(); + }; + let publishers = policy::publishers_of(&bundles, rule); + // NOBODY PUBLISHES IT, so there is nothing to scan FOR and the scan is + // skipped entirely rather than run and discarded. This is the honest + // answer for an engine-side rule name, which is a legitimate thing to + // mint against — `protected-mutation` has a real override route. + // + // Silent, deliberately: reporting the rule as unknown here would decide + // CLOUD-1551's open question about an anchor that binds `call:` when + // nothing matches, and this row must not settle that by accident. + if publishers.is_empty() { + return head(); + } + policy_rows + .into_iter() + .filter(|declared| publishers.contains(declared.id.as_str())) .collect() } else { exact diff --git a/crates/batten/src/policy.rs b/crates/batten/src/policy.rs index e90267895..195cc28d6 100644 --- a/crates/batten/src/policy.rs +++ b/crates/batten/src/policy.rs @@ -463,6 +463,36 @@ impl Bundle { } } +/// The enabling-row ids of every bundle that publishes `predicate` (CLOUD-1571). +/// +/// **The narrowing a mint needs, and the reason it is a function rather than a +/// filter written at its one call site.** [`crate::admission`]'s anchor has to +/// re-run the rule a refusal named in order to recover its fingerprint, and +/// `--rule` carries a PREDICATE id — `filed-here` publishes `filed-over-own-diff` +/// — so a row-id match selects nothing and the mint silently binds the head +/// (CLOUD-1087, CLOUD-1125). Widening from there to every `policy` row fixed that +/// and cost 2m22s per mint, because "which KIND of row" is not "which row": +/// `protected-mutation` is an engine-side name no bundle can publish, and 58 +/// modules were evaluated over the whole tree to produce findings the caller +/// discarded one line later. +/// +/// Asking each bundle's own published set answers the question exactly. An EMPTY +/// result is therefore a real answer — *nothing can raise this predicate, so +/// there is nothing to scan for* — and never a could-not-look: a caller reads it +/// as "skip the scan", which is only sound because [`Bundle::declared`] is the +/// same authority [`Bundle::attribute`] resolves a violation's id against. +/// +/// Borrowed rather than owned, so a caller filtering its own rows against the +/// result allocates nothing per row. +#[must_use] +pub fn publishers_of<'a>(bundles: &'a [Bundle], predicate: &str) -> BTreeSet<&'a str> { + bundles + .iter() + .filter(|bundle| bundle.declared().contains(predicate)) + .map(Bundle::id) + .collect() +} + impl std::fmt::Debug for Bundle { /// Names the row, its modules' paths and the ids they publish — and **never /// a source**, so a policy body cannot reach a log through a derived `Debug` diff --git a/crates/batten/tests/it/admission_narrowing.rs b/crates/batten/tests/it/admission_narrowing.rs new file mode 100644 index 000000000..e1360415b --- /dev/null +++ b/crates/batten/tests/it/admission_narrowing.rs @@ -0,0 +1,228 @@ +//! `policy::publishers_of` — the narrowing a mint's anchor selects with +//! (CLOUD-1571). +//! +//! # What went wrong, because it decides what these cases have to pin +//! +//! `admission_anchor` re-runs the rule a refusal named so it can recover that +//! finding's fingerprint and bind the admission to it. `--rule` carries a +//! PREDICATE id — `filed-here` publishes `filed-over-own-diff` — so filtering +//! `declared.id == rule` selected nothing, the scan produced no finding, and the +//! mint silently took the `head()` fallback: an admission answered, spent, and +//! queried by nothing (CLOUD-1087, CLOUD-1125). +//! +//! Widening an empty exact match to every `policy` row fixed that and cost +//! **2m22.121s per mint**, measured on `main` at `6eb08e14` against **0.123s** +//! for a full adjudication of the same tree. The bound "widens to those rows and +//! no further" is a statement about which KIND of row, and the regression is that +//! this is not the same as which ROW: `protected-mutation` is an engine-side rule +//! name with zero hits under `policy/`, so **no bundle could ever publish it**, +//! and 58 modules were evaluated over the whole tree to produce a finding set +//! discarded one line later. One test case paying that was 24% of the entire +//! suite. +//! +//! # Why the pure function is the subject +//! +//! The narrowing is the decision; the early return that skips the scan is one +//! line downstream of it. `admission_anchor` reads stdin and is reachable only +//! through `override request`, so the decision is extracted where it can be +//! driven directly rather than asserted through a verb that also parses answers, +//! resolves an epoch and reads a git identity. +//! +//! **The last case is the one that matters, and it is not a fixture.** It asks +//! the REAL committed bundles the real question, so a module later publishing +//! `protected-mutation` — which would silently restore the full scan — reddens +//! here. A fixture-only suite would pass over exactly that. + +// Panicking on setup failure is the idiomatic way for a test to fail loudly. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use crate::common; + +use std::fs; +use std::path::{Path, PathBuf}; + +use batten::policy::{self, Bundle}; +use batten::rules::Rule; + +/// A module publishing one predicate whose id is not its enabling row's — the +/// shape that defeated the `declared.id == rule` filter in the first place. +const PUBLISHES: &str = r#" +package batten + +import rego.v1 + +rules contains "PREDICATE" + +violation contains {"rule": "PREDICATE", "verdict": "stray key probe"} if { + input.tree.documents["config.toml"].stray +} +"#; + +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("batten-narrow-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("scratch"); + dir +} + +/// One bundle folder per row, each publishing a predicate the row is not named +/// after. Distinct packages and distinct predicate ids, because `load` refuses a +/// shared source and a shared id alike. +fn write_bundle(root: &Path, row_id: &str, predicate: &str) { + let dir = root.join(format!("policy-{row_id}")); + fs::create_dir_all(&dir).expect("bundle folder"); + let module = PUBLISHES + .replace("package batten", &format!("package batten.b{row_id}")) + .replace("PREDICATE", predicate); + fs::write(dir.join("gate.rego"), module).expect("module"); +} + +fn row(id: &str) -> Rule { + serde_json::from_value(serde_json::json!({ + "id": id, + "kind": "policy", + "scope": "tree", + "bundle": format!("policy-{id}/"), + "documents": ["config.toml"], + "severity": "deny", + })) + .expect("a tree-scoped policy row the loader accepts") +} + +/// The vocabulary derived from what the fixture's modules actually raise — +/// registry equality runs both ways, so a table naming an unraised token is dead +/// vocabulary and `load` refuses it. +fn load_fixture(root: &Path, rows: &[Rule]) -> Vec { + let verdicts = common::verdicts_in(root); + policy::load( + root, + rows, + policy::Vocabulary { + // NO PATTERN ROWS, and that is a statement rather than an omission: + // these modules resolve none, so supplying any would be input no + // consumer supplies. + patterns: &[], + verdicts: &verdicts, + recorders: &[], + }, + policy::ModuleChecks::RunOverSelection, + None, + ) + .expect("the fixture bundles load") +} + +/// THE REGRESSION, in one assertion. A predicate no bundle publishes selects no +/// bundle, so the caller has nothing to scan for and skips the scan entirely. +/// +/// Fails by: restoring the widen-to-every-policy-row filter, which selects both +/// rows here and, on the real tree, all 58. +#[test] +fn a_predicate_no_bundle_publishes_selects_nothing() { + let root = scratch("unpublished"); + write_bundle(&root, "alpha", "alpha-violated"); + write_bundle(&root, "beta", "beta-violated"); + let bundles = load_fixture(&root, &[row("alpha"), row("beta")]); + + assert!( + policy::publishers_of(&bundles, "protected-mutation").is_empty(), + "an engine-side rule name is published by nothing, so nothing is selected \ + and no module is evaluated (CLOUD-1571)" + ); +} + +/// THE PROPERTY CLOUD-1087 AND CLOUD-1125 BOUGHT, which this must not spend. A +/// predicate a bundle DOES publish still selects that bundle — so the mint still +/// reaches its finding and still anchors on it rather than falling back. +/// +/// Fails by: narrowing back to `declared.id == rule`, which selects nothing here +/// because no row is named after the predicate it publishes. +#[test] +fn a_published_predicate_selects_its_own_bundle_and_only_that_one() { + let root = scratch("published"); + write_bundle(&root, "alpha", "alpha-violated"); + write_bundle(&root, "beta", "beta-violated"); + let bundles = load_fixture(&root, &[row("alpha"), row("beta")]); + + let selected = policy::publishers_of(&bundles, "beta-violated"); + assert_eq!( + selected.into_iter().collect::>(), + vec!["beta"], + "exactly the publishing bundle, so the scan is one module rather than every one" + ); +} + +/// ANTI-VACUITY ON THE FIXTURE ITSELF. Both cases above would pass over a bundle +/// set that failed to publish anything at all — an empty `declared` set makes +/// every query empty, and the first case would then be green for the wrong +/// reason. +#[test] +fn the_fixture_bundles_actually_publish_their_predicates() { + let root = scratch("anti-vacuity"); + write_bundle(&root, "alpha", "alpha-violated"); + let bundles = load_fixture(&root, &[row("alpha")]); + + assert_eq!(bundles.len(), 1, "one row, one bundle"); + assert!( + bundles[0].declared().contains("alpha-violated"), + "the module publishes the predicate the cases above query for; without \ + this, an empty declared set would make every query vacuously empty" + ); + assert!( + !policy::publishers_of(&bundles, "alpha-violated").is_empty(), + "and the query reaches it" + ); +} + +/// THE FIDELITY CASE, over the REAL committed bundles rather than a fixture. +/// +/// `protected-mutation` is the measured instance: it is the rule every protected +/// path write mints against, it is engine-side (`decision.rs`, `refusal.rs`, +/// `hook.rs`, `rules.rs`, `admission.rs`), and it has **zero hits under +/// `policy/`**. If some later module publishes it, the scan silently goes wide +/// again and the 2m22s comes back with nothing to announce it. That is what this +/// case refuses, and no fixture can. +/// +/// The committed `filed-over-own-diff` is asserted beside it in the same +/// function, because a repository whose bundles failed to load would give an +/// empty set for the first assertion and pass it for exactly the wrong reason. +#[test] +fn the_committed_bundles_publish_no_engine_side_rule_name() { + let root = common::at_root("batten.toml"); + let root = root.parent().expect("the committed config has a parent"); + let config = batten::resolve::resolve(root, None).expect("the committed config resolves"); + let policy_rows: Vec<_> = config + .rules + .iter() + .filter(|declared| declared.kind == batten::rules::RuleKind::Policy) + .cloned() + .collect(); + let bundles = policy::load( + root, + &policy_rows, + policy::Vocabulary { + patterns: &config.patterns, + verdicts: &config.verdicts, + recorders: &config.recorders, + }, + policy::ModuleChecks::RunOverSelection, + None, + ) + .expect("the committed bundles load"); + + // ANTI-VACUITY FIRST, so the refusal below cannot pass over an empty set. + assert!( + policy::publishers_of(&bundles, "filed-over-own-diff") + .into_iter() + .eq(["filed-here"]), + "the committed tree still publishes a predicate under a differently-named \ + row — the shape the whole narrowing exists to handle" + ); + + assert!( + policy::publishers_of(&bundles, "protected-mutation").is_empty(), + "`protected-mutation` is engine-side and no module may publish it; if one \ + does, every mint against it goes back to evaluating {} bundles over the \ + whole tree (CLOUD-1571)", + bundles.len() + ); +} diff --git a/crates/batten/tests/it/main.rs b/crates/batten/tests/it/main.rs index a7c3c0a10..430057878 100644 --- a/crates/batten/tests/it/main.rs +++ b/crates/batten/tests/it/main.rs @@ -46,6 +46,7 @@ mod acquisition_sweep; mod address_resolve; mod address_transport; mod admission; +mod admission_narrowing; mod advisory_drain; mod agent_capabilities; mod agent_facts; From d19f9ab73967605905d127cd0f5f1ccb87b6f986 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Mon, 7 Sep 2026 20:19:03 +0000 Subject: [PATCH 02/13] fix(test): resolve the committed config through the Overrides the signature takes `resolve::resolve` takes `&Overrides`, not an `Option`. The fidelity case was written against the wrong arity and the tier could not compile, so none of the four cases ran. Refs: CLOUD-1571 --- crates/batten/tests/it/admission_narrowing.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/batten/tests/it/admission_narrowing.rs b/crates/batten/tests/it/admission_narrowing.rs index e1360415b..fcb64d342 100644 --- a/crates/batten/tests/it/admission_narrowing.rs +++ b/crates/batten/tests/it/admission_narrowing.rs @@ -189,7 +189,8 @@ fn the_fixture_bundles_actually_publish_their_predicates() { fn the_committed_bundles_publish_no_engine_side_rule_name() { let root = common::at_root("batten.toml"); let root = root.parent().expect("the committed config has a parent"); - let config = batten::resolve::resolve(root, None).expect("the committed config resolves"); + let config = batten::resolve::resolve(root, &batten::resolve::Overrides::default()) + .expect("the committed config resolves"); let policy_rows: Vec<_> = config .rules .iter() From 37da3ffdfbb11332216d50d19a15d8ededec2d1f Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 00:17:39 +0000 Subject: [PATCH 03/13] test(nextest): probe the suite with a report-only slow threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step one of ratcheting a slow-test ban into place. `slow-timeout` alone MARKS a case slow and prints it; without `terminate-after` nothing is killed, so this pass produces the list the real period is chosen from rather than reddening cases nobody has looked at yet. 1s is deliberately far below where the ratchet will start. Measured mid-run: 206 of 2,286 cases over 1s, ~9%. That is the reading that says 1s cannot be the day-one value — the period starts above today's outliers and steps down, each step forcing the newly-slow cases to be made fast or explicitly overridden. The ban itself is `terminate-after`, which marks a case TIMEOUT and treats it as a failure. It is armed in the next commit, once the period is chosen from a full run, and shown able to fail before it is trusted. Refs: CLOUD-1571 --- .config/nextest.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 8bdb48a06..387f21e22 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -31,6 +31,15 @@ # `files_read` and `bytes_read`, which count acquisitions at the boundary and so # cannot see an O(n*m) evaluation over a document already in hand. +# THE SLOW-TEST TERMINATOR (CLOUD-1571 follow-on). +# +# PROVISIONAL, REPORT-ONLY: no `terminate-after` yet, so nextest MARKS a case +# slow and prints it without killing it. This pass exists to produce the list the +# real setting is chosen from; arming it before measuring would redden cases +# nobody has looked at. +[profile.default] +slow-timeout = "1s" + [[profile.default.overrides]] filter = 'test(deleting_six_governed_paths_costs_a_flat_multiple_of_the_floor)' threads-required = "num-test-threads" From e960ed780140180f06878beac0a2026443af3ef3 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 00:28:56 +0000 Subject: [PATCH 04/13] feat(nextest): ban slow tests in the runner, and gate the declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner already measures every case and can fail the slow ones, so nothing here re-derives a duration. `.config/nextest.toml` declares `slow-timeout = { period = "10s", terminate-after = 9 }`: nextest marks a case slow at 10s and KILLS it at 90s, reporting TIMEOUT — a failure, since `on-timeout` defaults to "fail". Shown rather than asserted. Under a deliberately strict profile a case reported TERMINATING, then TIMEOUT, then `error: test run failed`, exit 100. A wrong `terminate-after` or `on-timeout` would have been byte-identical to green. The numbers come from a measured full run, not from a target. 5,021 cases, all passing, 987.5s summed against 255.5s wall: p50 42ms | p95 671ms | p99 2.53s | max 64.85s over 10s: 11 | over 20s: 5 | over 30s: 2 | over 60s: 1 So `period` names 11 cases every run — the visibility half and the number the ratchet walks down — while the kill sits above today's worst on purpose. These are wall times under parallelism and nextest bills time spent BLOCKED as well as running (CLOUD-1439), so a case clear of the threshold on a quiet box crosses it on a loaded one. A bound a green tree fails on a slower runner is measuring the runner. Each step down is its own reviewed change. WHY THE RUNNER RATHER THAN A GATE OF OUR OWN. A hand-rolled wall-clock assertion is the instrument this repository has already refused twice: suite-bench-check records that a duration gate "would be red on every second run and would be bypassed within a day", and CLOUD-1419 wrote an aggregate ratchet and withdrew it in the same branch because its first firing was on the change that improved the thing it guarded. nextest's timeout has neither problem — per-case, filtered per-case overrides, maintained upstream. `policy/nextest-slow.rego` therefore guards the DECLARATION, not the duration. Two predicates: no terminating declaration this gate can read (`suite bind missing`), and a period above the module's committed ceiling (`bound edit refused`). Lowering is free, which is what makes it a ratchet rather than an equality check — a branch that makes the suite faster never negotiates with the gate. The period is read with string builtins alone, because an inline regex is refused at load and this is not a concept the [[pattern]] registry should carry. A unit the module cannot convert REFUSES rather than passing: nextest accepts `2m` and `500ms`, either of which would otherwise leave the comparison unreachable and the gate silently green over a bound nobody enforces. The compiled tier's absent-file case is the channel probe. `landing-roster-guarded` shipped a could-not-look arm that could never fire, because its row named a GLOB and a glob matching zero files leaves the rule SKIPPED rather than evaluated. This row names a literal path, so the source is still acquired and the refusal still fires — measured over the engine rather than assumed, since "should" is what that module also believed. Verified: policy test 63 bundles, 801 passed; `batten check --rule nextest-slow` exit 0 over the committed tree. Refs: CLOUD-1571 Admits: f444dd47206c07939bdabb5857b1aa7364b678b13310a092ca27383dbb77c574 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: batten.toml Admits-anchor: call:33ffc6103de150e1907ea07f6e1ca8eca498f180 Admits-epoch: ed0cc67ec403a5849f97cbe66586e656ab21f7b6d2015725b2d51a2a2c6f5b91 Admits-author: alec@wenzowski.com Admits-prev: 602066abacece58b19a380d2b9751de501d0c9bc1eb25c4c3b1ed4ba23559250 Admits-answer-lost: The slow-test ban has no gate behind it. `.config/nextest.toml` now declares slow-timeout with terminate-after, which is what actually kills a slow case, but nothing refuses raising the period or dropping terminate-after — so the ban can be switched off in one line with no gate noticing. Registering `nextest-slow` plus its three verdict rows is what makes the declaration guarded, and rule 2 says a rule without a runnable gate is half a change. Admits-answer-precondition: The module and its ceiling are written and its own test tier passes; the compiled tier and MUTANT enrolment follow in the same PR. `config read first` cannot carry this: it points AT batten.toml, which is the file that must gain the row. `patch run first` is `git restore`, which discards the module. Admits-answer-rejected-route: `config read first` is a read route over the file that must be written. `patch run first` discards the work. Neither can add a [[rule]] row, and a registration is the only thing that makes a .rego module reachable by the engine. --- .config/nextest.toml | 42 ++++- batten.toml | 53 +++++++ crates/batten/tests/it/main.rs | 1 + crates/batten/tests/it/nextest_slow.rs | 208 ++++++++++++++++++++++++ mise.toml | 2 +- policy/nextest-slow.rego | 210 +++++++++++++++++++++++++ 6 files changed, 509 insertions(+), 7 deletions(-) create mode 100644 crates/batten/tests/it/nextest_slow.rs create mode 100644 policy/nextest-slow.rego diff --git a/.config/nextest.toml b/.config/nextest.toml index 387f21e22..ebd8e189d 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -31,14 +31,44 @@ # `files_read` and `bytes_read`, which count acquisitions at the boundary and so # cannot see an O(n*m) evaluation over a document already in hand. -# THE SLOW-TEST TERMINATOR (CLOUD-1571 follow-on). +# THE SLOW-TEST TERMINATOR, AND ITS RATCHET (CLOUD-1571 follow-on). # -# PROVISIONAL, REPORT-ONLY: no `terminate-after` yet, so nextest MARKS a case -# slow and prints it without killing it. This pass exists to produce the list the -# real setting is chosen from; arming it before measuring would redden cases -# nobody has looked at. +# `slow-timeout.period` is when nextest MARKS a case slow and prints it; +# `period x terminate-after` is when it KILLS it, reporting TIMEOUT — and a +# TIMEOUT is a FAILURE, because `on-timeout` defaults to "fail". So this pair is +# the ban on slow tests, enforced by the runner on every local run and in CI. +# Nothing in batten re-derives the measurement; `policy/nextest-slow.rego` only +# refuses weakening what is written here. +# +# WHY THE RUNNER RATHER THAN A GATE OF OUR OWN. A hand-rolled wall-clock +# assertion is the instrument this repository has already refused twice: +# `mise-tasks/suite-bench-check.sh` records that a duration gate "would be red on +# every second run and would be bypassed within a day", and CLOUD-1419 wrote an +# aggregate ratchet and withdrew it in the same branch because its first firing +# was on the change that improved the thing it guarded. nextest's own timeout has +# neither problem: it is per-case, it carries filtered per-case overrides, and it +# is maintained upstream. +# +# THE NUMBERS COME FROM A MEASURED FULL RUN, not from a target. 5,021 cases, all +# passing, summed 987.5s against 255.5s wall: +# +# p50 42ms | p95 671ms | p99 2.53s | max 64.85s +# over 10s: 11 cases | over 20s: 5 | over 30s: 2 | over 60s: 1 +# +# `period = "10s"` therefore names 11 cases on every run — the visibility half, +# and the number the ratchet walks down. `terminate-after = 9` kills at 90s, +# ~1.4x today's worst (64.85s, `agentic_record::a_replay_over_the_committed_ +# records_fires_on_every_required_key`). +# +# THE KILL THRESHOLD IS DELIBERATELY NOT SET AT TODAY'S WORST, and that is why +# this first step does not redden the tree. These are wall times under +# parallelism, and nextest bills a case for time spent BLOCKED as well as running +# (CLOUD-1439) — so a case near the threshold on a quiet box crosses it on a +# loaded one. A bound a green tree fails on a slower runner is measuring the +# runner. Each ratchet step down is its own reviewed change, and a step that +# strands a case gets that case fixed or an override naming why. [profile.default] -slow-timeout = "1s" +slow-timeout = { period = "10s", terminate-after = 9 } [[profile.default.overrides]] filter = 'test(deleting_six_governed_paths_costs_a_flat_multiple_of_the_floor)' diff --git a/batten.toml b/batten.toml index 3aada79e3..ede7df5a7 100644 --- a/batten.toml +++ b/batten.toml @@ -6331,6 +6331,35 @@ tool = "hyperfine" version = "1.20.0" input = "target/release/batten" +# The slow-test ban's declaration, guarded (CLOUD-1571 follow-on). +# +# THE BAN ITSELF IS THE RUNNER'S. `.config/nextest.toml`'s `slow-timeout` marks a +# case slow at `period` and kills it at `period x terminate-after`, reporting +# TIMEOUT — a failure, since `on-timeout` defaults to "fail". Measured on this +# tree: a case under a 50ms period reported TERMINATING, then TIMEOUT, then +# `error: test run failed`, exit 100. +# +# SO THIS ROW GUARDS A DECLARATION RATHER THAN A DURATION, and that division is +# deliberate. `mise-tasks/suite-bench-check.sh` records why a duration gate is the +# wrong instrument — it "would be red on every second run and would be bypassed +# within a day" — and CLOUD-1419 wrote an aggregate ratchet and withdrew it in the +# same branch. nextest measures and decides; batten refuses the weakening. +# +# `lines` NAMES ONE FILE AND THAT IS SAFE HERE, unlike `landing-roster-guarded` +# which had to declare a whole directory. That module's absent-source arm was dead +# because a glob matching zero files leaves the rule SKIPPED rather than evaluated +# — but `.config/nextest.toml` is a literal path, not a glob, so a tree without it +# still acquires the declared source and `nextest-slow-unbounded` fires on the +# empty document. `crates/batten/tests/it/nextest_slow.rs` drives the engine over a +# tree with the file deleted, because that is the only tier that can tell. +[[rule]] +id = "nextest-slow" +kind = "policy" +scope = "tree" +lines = [".config/nextest.toml"] +module = "policy/nextest-slow.rego" +severity = "deny" + # The agentic trials protocol (CLOUD-1116). A completeness predicate over the two # records in `bench/agentic/`, and nothing else — it decides whether a row that # CLAIMS an outcome carries the arms, the fixture, the run count and the falsifier @@ -12535,6 +12564,30 @@ id = "module read first" kind = "document" target = "policy/ci-cache-declared.rego" +[[verdict]] +id = "suite bind missing" +gloss = "the runner config declares no slow-timeout this gate can read, so no slow-test ban is in force" +class = """ +`period` alone only MARKS a case slow and prints it; `terminate-after` is what kills it, and the TIMEOUT that follows is what fails the run. A declaration carrying one and not the other is not a ban and must not read as one. An absent file, a commented-out declaration, and a period in a unit this module cannot convert all reach the same class deliberately: nextest accepts `2m` and `500ms`, either of which would leave the comparison unreachable and the gate silently green over a bound nobody is enforcing. Fail-closed is the only honest direction for a guard whose subject is whether a guard exists. +""" + +[[verdict.route]] +id = "module read first" +kind = "document" +target = "policy/nextest-slow.rego" + +[[verdict]] +id = "bound edit refused" +gloss = "the declared slow-test period is above the ceiling policy/nextest-slow.rego commits to" +class = """ +The ratchet is "never above", not "always exactly", and the asymmetry is the point: lowering the period is free, so a branch that makes the suite faster tightens the bound without negotiating with this gate. Raising it past the ceiling is refused. Lowering the CEILING is a reviewed edit to the module, which is where a reader sees it in the diff rather than inferring it from a number that moved. +""" + +[[verdict.route]] +id = "module read first" +kind = "document" +target = "policy/nextest-slow.rego" + [[startup]] id = "engine-reads-the-authority" gloss = "the engine that hook registrations invoke by bare name can read this repository's committed config" diff --git a/crates/batten/tests/it/main.rs b/crates/batten/tests/it/main.rs index 430057878..3e0f9f2f5 100644 --- a/crates/batten/tests/it/main.rs +++ b/crates/batten/tests/it/main.rs @@ -169,6 +169,7 @@ mod mise_pin_agreement; mod mutate; mod mutation_declared_case; mod narrow_adoption; +mod nextest_slow; mod obligations_bound; mod outcome_advice; mod perf_assert; diff --git a/crates/batten/tests/it/nextest_slow.rs b/crates/batten/tests/it/nextest_slow.rs new file mode 100644 index 000000000..dd9615200 --- /dev/null +++ b/crates/batten/tests/it/nextest_slow.rs @@ -0,0 +1,208 @@ +//! `policy/nextest-slow.rego` over the COMPILED engine (CLOUD-1571 follow-on). +//! +//! # Why this file exists when the module already has `test_` rules +//! +//! Those are the load-time tier and they pin the PREDICATE. They cannot pin that +//! the engine BUILDS the input the predicate reads: `with input as` fabricates +//! the very shape the engine may be unable to produce, so a module reading a key +//! nothing fills passes its own suite green and enforces nothing. +//! +//! # The absent-file case is the one that earns this file +//! +//! `landing-roster-guarded` shipped with a could-not-look arm that could never +//! fire: its row named a GLOB, and a glob matching zero files leaves the rule +//! SKIPPED rather than evaluated over an empty document, so a branch deleting the +//! subject passed clean. `.config/nextest.toml` is a literal path rather than a +//! glob, which should mean the declared source is still acquired and the refusal +//! still fires — but "should" is exactly what that module also believed. +//! `an_absent_runner_config_is_refused` is what decides it, and only a tier +//! driving the engine over a tree with the file removed can. +//! +//! It doubles as the channel probe `rules/policy-modules.md` prescribes: a clean +//! report and a module that never ran are byte-identical on the decision surface, +//! and a case that reddens when the subject is removed tells them apart. +//! +//! # And the fidelity case drives the REAL committed file +//! +//! `the_committed_config_declares_a_terminating_slow_timeout` scans this +//! repository's own `.config/nextest.toml`. A fixture-only suite would pass over +//! a tree whose slow-test ban had been quietly disarmed, which is the regression +//! this module exists to refuse. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use crate::common; + +use std::fs; +use std::path::{Path, PathBuf}; + +use batten::rules::{self, Rule}; + +/// The predicate ids the module declares. +const UNBOUNDED: &str = "nextest-slow-unbounded"; +const RAISED: &str = "nextest-slow-raised"; + +/// The one path the module is anchored on. +const CONFIG: &str = ".config/nextest.toml"; + +/// The committed shape: marks at 10s, kills at 90s. +const ARMED: &str = "[profile.default]\nslow-timeout = { period = \"10s\", terminate-after = 9 }\n"; + +/// `period` with no `terminate-after` only REPORTS. Not a ban. +const REPORT_ONLY: &str = "[profile.default]\nslow-timeout = \"10s\"\n"; + +/// Above the ceiling the module commits to. +const RAISED_BODY: &str = + "[profile.default]\nslow-timeout = { period = \"30s\", terminate-after = 3 }\n"; + +/// A fixture tree carrying a runner config with `body`, or none at all when +/// `body` is `None`. +fn repo(name: &str, body: Option<&str>) -> PathBuf { + let root = common::scratch(name); + fs::create_dir_all(root.join(".config")).expect("scratch config dir"); + if let Some(body) = body { + fs::write(root.join(CONFIG), body).expect("write the runner config"); + } + install_module(&root); + root +} + +/// The COMMITTED module, copied rather than re-typed. A fixture carrying its own +/// copy of the predicate would pass while the shipped one was broken, which is +/// the fidelity failure this tier exists to catch. +fn install_module(root: &Path) { + let source = common::at_root("policy/nextest-slow.rego") + .canonicalize() + .expect("the committed module is where the row says it is"); + fs::create_dir_all(root.join("policy")).expect("scratch policy dir"); + fs::copy(source, root.join("policy/nextest-slow.rego")).expect("install committed module"); +} + +/// The committed row's shape, so a registration the loader would reject cannot +/// pass here. +fn row() -> Rule { + serde_json::from_value(serde_json::json!({ + "id": "nextest-slow", + "kind": "policy", + "scope": "tree", + "lines": [CONFIG], + "module": "policy/nextest-slow.rego", + "severity": "deny", + })) + .expect("the loader accepts the committed row's shape") +} + +fn scan(root: &Path) -> rules::Scan { + let verdicts = common::verdicts_in(root); + // NO PATTERN ROWS, and that is a statement rather than an omission: this + // module resolves none — the period is read with string builtins precisely so + // no inline regex and no registry row is needed — so supplying any would be + // input no consumer supplies and the cases would pass for the wrong reason. + rules::run_static( + &[row()], + &[], + batten::policy::Vocabulary { + patterns: &[], + verdicts: &verdicts, + recorders: &[], + }, + root, + ) + .expect("the read surface runs a policy row") +} + +fn rules_fired(root: &Path) -> Vec { + scan(root) + .findings + .into_iter() + .map(|finding| finding.rule) + .collect() +} + +/// THE FIDELITY CASE. The COMMITTED bytes of the file that actually bans slow +/// tests here, scanned by the engine in a fixture — the repo root itself cannot be +/// scanned with a one-rule subset, because `check_registry_is_exhausted` then +/// reports every class the other rules would have raised. +/// `#MUTANT declaration-unread` reddens exactly here. +#[test] +fn the_committed_config_declares_a_terminating_slow_timeout() { + let committed = fs::read_to_string(common::at_root(CONFIG)) + .expect("the committed runner config is where the row says it is"); + let root = repo("nextest-slow-committed", Some(&committed)); + assert!( + rules_fired(&root).is_empty(), + "the committed {CONFIG} must declare a slow-timeout with terminate-after, at or \ + under the module's ceiling; if this fails the slow-test ban has been disarmed" + ); +} + +/// THE CASE THIS TIER EXISTS FOR, and the one a `with input as` case cannot +/// decide. `landing-roster-guarded` carried a could-not-look arm that never fired +/// because its row named a glob and a glob matching nothing leaves the rule +/// SKIPPED. This row names a literal path; that it therefore still acquires the +/// source and still refuses is measured here rather than assumed. +/// +/// It is also the channel probe: a clean report and a module that never ran are +/// byte-identical, and this reddening is what tells them apart. +#[test] +fn an_absent_runner_config_is_refused() { + let root = repo("nextest-slow-absent", None); + assert_eq!( + rules_fired(&root), + vec![UNBOUNDED.to_owned()], + "deleting the runner config removes the ban, and must refuse rather than read clean" + ); +} + +/// `period` ALONE ONLY REPORTS. Without `terminate-after` nextest marks a case +/// slow and lets it run, so this is not a ban and must not read as one. +#[test] +fn a_declaration_without_terminate_after_is_refused() { + let root = repo("nextest-slow-report-only", Some(REPORT_ONLY)); + assert_eq!(rules_fired(&root), vec![UNBOUNDED.to_owned()]); +} + +/// THE RATCHET, over the engine. A period above the committed ceiling is refused. +#[test] +fn a_period_above_the_ceiling_is_refused() { + let root = repo("nextest-slow-raised", Some(RAISED_BODY)); + assert_eq!(rules_fired(&root), vec![RAISED.to_owned()]); +} + +/// AND LOWERING IS FREE — the asymmetry that makes this a ratchet rather than an +/// equality check. Without this case the rule above is satisfied by a module that +/// refuses every value that is not exactly the ceiling, which would make every +/// speed-up negotiate with the gate. +#[test] +fn a_period_below_the_ceiling_is_clean() { + let root = repo( + "nextest-slow-lowered", + Some("[profile.default]\nslow-timeout = { period = \"4s\", terminate-after = 9 }\n"), + ); + assert!(rules_fired(&root).is_empty()); +} + +/// A UNIT THE MODULE CANNOT CONVERT REFUSES rather than leaving the comparison +/// unreachable. `2m` is a legal nextest value, and without this arm it would read +/// as a clean tree while no bound was being enforced at all — the silent hole the +/// module's header calls fail-closed. +#[test] +fn a_period_in_an_unconvertible_unit_is_refused() { + let root = repo( + "nextest-slow-minutes", + Some("[profile.default]\nslow-timeout = { period = \"2m\", terminate-after = 3 }\n"), + ); + assert_eq!(rules_fired(&root), vec![UNBOUNDED.to_owned()]); +} + +/// AND A COMMENTED-OUT DECLARATION IS NOT ONE. The ordinary shape of "this was +/// flaky, disabling it for now" leaves the text in the file, which is exactly how +/// `landing-roster-guarded`'s first draft was defeated by its own documentation. +#[test] +fn a_commented_declaration_does_not_arm_the_ban() { + let root = repo( + "nextest-slow-commented", + Some("[profile.default]\n# slow-timeout = { period = \"10s\", terminate-after = 9 }\n"), + ); + assert_eq!(rules_fired(&root), vec![UNBOUNDED.to_owned()]); +} diff --git a/mise.toml b/mise.toml index 211e87907..2407c3b9e 100644 --- a/mise.toml +++ b/mise.toml @@ -478,7 +478,7 @@ CI_FANIN_WORKFLOW = ".github/workflows/ci.yml" # which is a property of the world and belongs on a clock (`lock-complete`). REGORUS_OPA_COMPLIANCE = "1.2.0" REGORUS_OPA_COMPLIANCE_FOR = "0.11" -MUTANT_GATES = "mise,attestation-check,engine-config,engine-doctor,engine-landed,engine-pinned,engine-ready,agentic-experiment-record,awk-regex-check,bats-invocation,batten-glob-check,board-diff-overlap,board-payloads,board-sweep,branch-age-check,cap-drift,ci-cache-declared,ci-hygiene,ci-lease-precondition,ci-parity,ci-slow-needed,ci-suite-lane,ci-tools-check,claim-before-code,claim-order-is-stated,claimed-keys,closing-key-check,coderabbit-config-check,commit-hygiene,connector-allow-guard,connector-allow-resolve,container-preflight,darwin-link,deferral-check,denials-outlive-the-turn,digest-major-agreement,doctor,done-check,done-pr-check,duplicate-close-check,egress-fencing,evaluator-closure-check,evaluator-io-check,filed-here,finding-sink-check,fixture-forks,forge-verdict-required,graph-check,harness-grant,harness-wiring,hk-fix-selection,hk-plan-required,hook-pin-check,hook-skip-local,in-progress-drain,install-check,land,land-divergence-assert,land-lock,land-lock-check,landed-check,landing-loop,landing-roster-guarded,leased-push,license-table-check,linear-check,lock-complete,macos-link-check,mcp-allow-check,mcp-attach-check,mcp-timeout-budget,merged-pr-keys,mise-action-floor,mise-pin-agreement,module-map-check,msrv-pin-agreement,mutation-declared-case,no-doctests,nonverdict-assert,ntia-check,obligations-bound,perf-assert,pinned-toolchain,pipefail-grep-check,plan-complete,pr-partition-restated,pr-unsubscribed,privileged-lane,prose-only,publish-credential-check,ready-cites-check,ready-lint,reclaim-census,release-assets-check,release-due,release-provision-parity,release-tag-shape,release-tracking-check,released,remedy-authorship,repetition-without-progress,report-only-check,review-answered,review-dispatched,run-shape,rust-paths-check,sbom,sbom-inventory,serena-mcp,shell-hygiene,shell-retirement,shell-write-advisory,signing-posture,sonar-gate,spec-ref-check,stop-posture,stop-posture-check,suite-bench-check,suite-subject-retirable,task-substitution,test-targets,timeout-check,token-bench-check,transcript-corpus-check,tree-clean,trunk-based,validator-verdict-clean,verdict-routes-resolve,verified,weakens-declared,worktree-registration" +MUTANT_GATES = "mise,attestation-check,engine-config,engine-doctor,engine-landed,engine-pinned,engine-ready,agentic-experiment-record,awk-regex-check,bats-invocation,batten-glob-check,board-diff-overlap,board-payloads,board-sweep,branch-age-check,cap-drift,ci-cache-declared,ci-hygiene,ci-lease-precondition,ci-parity,ci-slow-needed,ci-suite-lane,ci-tools-check,claim-before-code,claim-order-is-stated,claimed-keys,closing-key-check,coderabbit-config-check,commit-hygiene,connector-allow-guard,connector-allow-resolve,container-preflight,darwin-link,deferral-check,denials-outlive-the-turn,digest-major-agreement,doctor,done-check,done-pr-check,duplicate-close-check,egress-fencing,evaluator-closure-check,evaluator-io-check,filed-here,finding-sink-check,fixture-forks,forge-verdict-required,graph-check,harness-grant,harness-wiring,hk-fix-selection,hk-plan-required,hook-pin-check,hook-skip-local,in-progress-drain,install-check,land,land-divergence-assert,land-lock,land-lock-check,landed-check,landing-loop,landing-roster-guarded,leased-push,license-table-check,linear-check,lock-complete,macos-link-check,mcp-allow-check,mcp-attach-check,mcp-timeout-budget,merged-pr-keys,mise-action-floor,mise-pin-agreement,module-map-check,msrv-pin-agreement,mutation-declared-case,no-doctests,nonverdict-assert,ntia-check,obligations-bound,perf-assert,pinned-toolchain,pipefail-grep-check,plan-complete,pr-partition-restated,pr-unsubscribed,privileged-lane,prose-only,publish-credential-check,ready-cites-check,ready-lint,reclaim-census,release-assets-check,release-due,release-provision-parity,release-tag-shape,release-tracking-check,released,remedy-authorship,repetition-without-progress,report-only-check,review-answered,review-dispatched,run-shape,rust-paths-check,sbom,sbom-inventory,serena-mcp,shell-hygiene,shell-retirement,shell-write-advisory,signing-posture,sonar-gate,spec-ref-check,stop-posture,stop-posture-check,suite-bench-check,suite-subject-retirable,task-substitution,test-targets,timeout-check,token-bench-check,transcript-corpus-check,tree-clean,trunk-based,validator-verdict-clean,verdict-routes-resolve,verified,weakens-declared,worktree-registration,nextest-slow" # --- GitHub reachability behind an egress proxy (Claude Code web sandbox etc.) --- # mise resolves every tool's release through GitHub's *API* host, api.github.com. diff --git a/policy/nextest-slow.rego b/policy/nextest-slow.rego new file mode 100644 index 000000000..f490de5a8 --- /dev/null +++ b/policy/nextest-slow.rego @@ -0,0 +1,210 @@ +# The slow-test ban, and the ratchet that walks it down (CLOUD-1571 follow-on). +# +# WHAT ENFORCES THE BAN IS NOT THIS MODULE. `.config/nextest.toml` declares +# `slow-timeout = { period, terminate-after }`; nextest marks a case slow at +# `period` and KILLS it at `period x terminate-after`, reporting TIMEOUT, and a +# TIMEOUT is a failure because `on-timeout` defaults to "fail". That is the ban, +# in the runner, on every local run and in CI. Measured on this tree before this +# module was written: a case run under a 50ms period reported `TERMINATING`, then +# `TIMEOUT`, then `error: test run failed`, exit 100. +# +# THIS MODULE ONLY REFUSES WEAKENING IT. The split matters. A gate that +# re-derived per-case durations would be the instrument this repository has +# already refused twice — `mise-tasks/suite-bench-check.sh` records that a +# duration gate "would be red on every second run and would be bypassed within a +# day", and CLOUD-1419 wrote an aggregate ratchet and withdrew it in the same +# branch because its first firing was on the change that improved the thing it +# guarded. The runner's timeout has neither problem, so the honest division is: +# nextest measures and decides, batten guards the declaration. +# +# THE CEILING IS A LITERAL HERE, exactly as `perf-assert.rego` holds its budgets. +# A `policy/*.rego` module IS consumer config, so a number is at home in it; +# non-negotiable rule 1 scopes to `crates/batten`. `rules/policy-modules.md` +# refuses a threshold spelled as a `[[pattern]]` row for the opposite reason — +# arithmetic is not a concept with one spelling — and this is not that. +# +# THE RATCHET IS "NEVER ABOVE", NOT "ALWAYS EXACTLY". `ceiling_seconds` is the +# committed maximum. Lowering the period below it is free, which is the whole +# point: a branch that makes the suite faster tightens the bound without +# negotiating with this gate. Raising it above the ceiling is refused, and +# lowering the CEILING is a reviewed edit to this file that a reader sees in the +# diff. That asymmetry is what a ratchet is. +# +# NO INLINE REGEX, AND THE PARSE IS STRING BUILTINS ONLY. An inline pattern is +# refused at load and this is not a concept the `[[pattern]]` registry should +# carry, so the period is read by splitting the line on the quote character. +# `slow-timeout = { period = "10s", ... }` splits into three parts and part 1 is +# `10s`; trimming the `s` and converting is the whole parse. +# +# AND AN UNREADABLE PERIOD REFUSES RATHER THAN PASSING, which is the direction +# that matters. nextest accepts `2m` and `500ms` as well as `10s`; both would +# leave `to_number` undefined, the comparison unreachable, and the gate silently +# green over a bound nobody is enforcing. So the absent, the unterminated and the +# unreadable are ONE class — `slow bound missing` — and every one of them means +# the same thing: no ban is in force that this gate can vouch for. +#MUTANT-SUITE crates/batten/tests/it/nextest_slow.rs +#MUTANT terminator-unread|s@^\tcontains(line, "terminate-after")$@\ttrue@|a_period_without_terminate_after_is_refused +#MUTANT ceiling-may-rise|s@^\tperiod > ceiling_seconds$@\tfalse@|a_period_above_the_ceiling_is_refused +#MUTANT declaration-unread|s@^\tsome line in input.tree.lines\[config\]$@\tsome line in []@|the_committed_config_declares_a_terminating_slow_timeout +# +# THE THIRD MUTATION EMPTIES THE LINE WALK rather than negating a conjunct, for +# `landing-roster-guarded`'s reason: emptying it makes the declaration +# unreachable, which reddens the PASS case over the real committed file. The +# first two redden refusal cases. So the three reach different cases and none of +# them is shadowed by another. + +# METADATA +# description: | +# Bound to the TREE surface: this row is `scope = "tree"`, so it reads +# `input.tree` and never the mediated call. +# THIS BLOCK IS YAML AND MUST STAY THE LAST COMMENT BLOCK BEFORE `package`. +# schemas: +# - input: schema["policy-input.schema"] +package batten.nextest_slow + +import rego.v1 + +rules contains "nextest-slow-unbounded" + +rules contains "nextest-slow-raised" + +# The runner's committed configuration. A consumer path in a consumer module, +# which is where non-negotiable rule 1 puts it. +config := ".config/nextest.toml" + +# The ceiling, in seconds: the largest `period` this repository will accept. +# +# Walked down as the suite gets faster, one reviewed step at a time. It sits at +# today's declared value, so the gate is exactly as tight as the tree already is +# and its first firing can only be on a change that loosens the bound — never on +# the tree it inherits, which is the shape `fixture-forks.rego` records as the one +# that gets an exception written for it "and the exception is what rots". +ceiling_seconds := 10 + +# Every non-comment line of the committed runner config. +declaration contains line if { + some line in input.tree.lines[config] + not startswith(trim_space(line), "#") +} + +# A line that declares BOTH halves of the ban. `period` alone only reports; it is +# `terminate-after` that kills, so a declaration carrying one and not the other +# is not a ban and must not read as one. +terminating contains line if { + some line in declaration + contains(line, "slow-timeout") + contains(line, "period") + contains(line, "terminate-after") +} + +# The declared period in whole seconds, read with string builtins alone. +# +# Undefined where the value is not `s` — which is deliberate and is what +# the `nextest-slow-unbounded` arm below turns into a refusal, rather than +# letting `2m` or `500ms` leave the comparison unreachable and the gate green. +period_seconds contains seconds if { + some line in terminating + parts := split(line, "\"") + count(parts) > 1 + value := parts[1] + endswith(value, "s") + not endswith(value, "ms") + seconds := to_number(trim_suffix(value, "s")) +} + +# NO TERMINATING DECLARATION THIS GATE CAN READ. +# +# Absent, comment-only, missing `terminate-after`, or carrying a period in a unit +# this module cannot convert — one class, because every one of them means no ban +# is in force that can be vouched for. Refusing on an unreadable unit is the +# fail-closed direction: the alternative is a silently unreachable comparison. +violation contains { + "rule": "nextest-slow-unbounded", + "verdict": "suite bind missing", + "subjects": [{"path": config}], +} if { + count(period_seconds) == 0 +} + +# THE RATCHET. A declared period above the committed ceiling is refused; below it +# is free, so making the suite faster never has to negotiate with this gate. +violation contains { + "rule": "nextest-slow-raised", + "verdict": "bound edit refused", + "subjects": [{"path": config}], +} if { + some period in period_seconds + period > ceiling_seconds +} + +deny contains finding if { + some finding in violation +} + +# --- the module's own tier --------------------------------------------------- +# +# These pin the PREDICATE. They cannot pin that the ENGINE builds the input the +# predicate reads — `with input as` fabricates the very shape the engine may be +# unable to produce — so `crates/batten/tests/it/nextest_slow.rs` runs the same +# questions over the compiled binary against the real committed file. Both tiers, +# per `rules/policy-modules.md`, and the second is not optional. + +tree(lines) := {"tree": {"lines": lines}} + +armed := ["[profile.default]", "slow-timeout = { period = \"10s\", terminate-after = 9 }"] + +report_only := ["[profile.default]", "slow-timeout = \"10s\""] + +raised := ["[profile.default]", "slow-timeout = { period = \"30s\", terminate-after = 3 }"] + +minutes := ["[profile.default]", "slow-timeout = { period = \"2m\", terminate-after = 3 }"] + +commented := ["[profile.default]", "# slow-timeout = { period = \"10s\", terminate-after = 9 }"] + +# THE PASS SIDE FIRST: without it every refusal below is satisfied by a module +# that refuses everything. +test_an_armed_declaration_at_the_ceiling_is_clean if { + count(violation) == 0 with input as tree({".config/nextest.toml": armed}) +} + +# LOWERING IS FREE, which is the ratchet's whole asymmetry. +test_a_period_below_the_ceiling_is_clean if { + count(violation) == 0 with input as tree({".config/nextest.toml": [ + "[profile.default]", + "slow-timeout = { period = \"5s\", terminate-after = 9 }", + ]}) +} + +# `period` ALONE ONLY REPORTS. A declaration that marks a case slow and never +# kills it is not a ban, and must not read as one. +test_a_period_without_terminate_after_is_refused if { + count(violation) == 1 with input as tree({".config/nextest.toml": report_only}) +} + +test_a_period_above_the_ceiling_is_refused if { + count(violation) == 1 with input as tree({".config/nextest.toml": raised}) +} + +# A UNIT THIS MODULE CANNOT CONVERT REFUSES rather than leaving the comparison +# unreachable. `2m` is a legal nextest value and a silent hole without this. +test_a_period_in_an_unconvertible_unit_is_refused if { + count(violation) == 1 with input as tree({".config/nextest.toml": minutes}) +} + +# A COMMENTED-OUT DECLARATION IS NOT A DECLARATION. +test_a_commented_declaration_does_not_arm_the_ban if { + count(violation) == 1 with input as tree({".config/nextest.toml": commented}) +} + +# ABSENCE IS THE SAME CLASS: a tree with no runner config has no ban in force. +test_an_absent_config_is_refused if { + count(violation) == 1 with input as tree({}) +} + +# THE POINTER IS THE FILE a reader opens (rule 4), and the class is the one the +# registry declares for it. +test_the_refusal_points_at_the_runner_config if { + some v in violation with input as tree({".config/nextest.toml": raised}) + v.subjects[0].path == ".config/nextest.toml" + v.verdict == "bound edit refused" +} From 74272fba5785846b262e0c74d97f5335ceb148df Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 00:35:17 +0000 Subject: [PATCH 05/13] fix(prune): re-measure the warm floor against the tree it defends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `verify` refused with "not enough disk to run the gate" while **13809 MB were free against a 10839 MB floor**. There was ample disk. What refused was `[prune.warm.basis]`'s staleness arm — declared 220, live 232, tolerance 10 — and `verify` narrates that exit as a disk shortage. The misnaming is filed separately; this commit fixes the cause it was hiding. And the floor was genuinely stale, which is the part worth having. 9472 is 45.54 x 208 — the figure for the basis BEFORE the 2026-09-06 move to 220 — so warm has been budgeting for 208 stems against a tree of 232. The table says what that costs in its own words: "a floor taken against a smaller stem count passes and then lets the build write more than it budgeted for", arriving as a rustc IO error inside a test run rather than as a disk fault. The engine had already noticed and was judging against a LEARNED floor of 10839 MB rather than the declared 9472. Re-measured by the method the table prescribes rather than scaled: `du -sm target` immediately after a successful prune on this container, **11140 MB at 232 stems**. That is 48.02 MB per stem against the previous basis's 45.54, so the `keep x stems x size` model holds. Both bases move together. The 2026-09-06 entry records what happens otherwise: refreshing one and not the other made the very next lap refuse on the other arm with warm never breached, because the two arms are judged at different times and only one of them waits. `[prune.cold] mb` does NOT move, obeying the 2026-09-05 entry rather than ignoring it. Its exact measurement needs a build from an empty `target`, which needs more free space than this container has — it cannot be taken here, which is different from being skipped, and it stays owed. Confirmed: `mise run target-prune` now passes against the declared 11140 MB floor with no staleness arm, where before it refused. Refs: CLOUD-205 Admits: 52ef35e11c567975022494bd6920a6de13ab56d01f48b50dd17d3a14997a8d8d Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: batten.toml Admits-anchor: call:8f1a99a5f8af82b87005d970cb2c598441641ef7 Admits-epoch: 9aa0951f97d071f5038a860cc58b02bb4a57cd0991050ebaf82aaa415ea2c746 Admits-author: alec@wenzowski.com Admits-prev: c39b95a7f88de3447ce342e3cf509d5d359c884bc9d541f9e45fecca0c5ed59b Admits-answer-lost: verify cannot run: target-prune refuses on [prune.warm.basis] staleness (declared 220, live 232, tolerance 10) and verify narrates that exit as "not enough disk" when 13809MB were free against a 10839MB floor. Until the basis and floor move together, no verify receipt can be written and this branch cannot be readied. The floor is also genuinely stale: it was 9472 = 45.54 x 208, left behind when the basis moved to 220, so it budgets for fewer stems than the tree has — which the table itself says "lets the build write more than it budgeted for" and arrives as a rustc IO error inside a test run. Admits-answer-precondition: Measured on this container by the method the table prescribes: du -sm target immediately after a successful prune, 11140MB at 232 stems = 48.0MB/stem, against the previous model 45.54. Both bases move together because the 2026-09-06 entry records that moving one and not the other makes the next lap refuse on the other arm. [prune.cold] mb does not move: the table states its exact measurement needs a build from an empty target that this container cannot host, and that it is knowingly under-budgeted rather than unmeasured by oversight. Admits-answer-rejected-route: `config read first` points at batten.toml, the file that must be written. `patch run first` is git restore, which discards the whole branch. Neither can move a floor, and the floor is the thing refusing verify. Admits: dd3ae2d57b6c5dafe8f18fa361d30eb8b36402304ff2c9a0db29a2e3fb1cac00 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: batten.toml Admits-anchor: call:8f1a99a5f8af82b87005d970cb2c598441641ef7 Admits-epoch: 2c150c5f7a02ac6d7acb72449673e91578cea40834644454a2e85b8c40a290b6 Admits-author: alec@wenzowski.com Admits-prev: 52ef35e11c567975022494bd6920a6de13ab56d01f48b50dd17d3a14997a8d8d Admits-answer-lost: The floor moved to 11140 but both basis counts are still 220 against a live 232, so target-prune still refuses on the staleness arm and verify still writes no receipt. The config is mid-move: a floor re-measured at 232 stems paired with a basis declaring 220 is exactly the mismatch the table warns produces "a floor taken against a smaller stem count". Leaving it here is worse than either end state. Admits-answer-precondition: The warm floor is already moved to 11140/11140 measured 2026-09-08 by the previous admission, from du -sm target after a successful prune at 232 stems. This write only moves both basis counts to match it and records the dated entry the table conventionally carries for every move. Cold mb is deliberately untouched. Admits-answer-rejected-route: `config read first` points at the file that must be written. `patch run first` is git restore, which would discard the floor move just made and the whole branch with it. This is the third time this session an admission being one-shot per write has split a single logical config change, which is CLOUD-1579 measured again. --- batten.toml | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/batten.toml b/batten.toml index ede7df5a7..3f4d12f5b 100644 --- a/batten.toml +++ b/batten.toml @@ -7371,10 +7371,10 @@ root = "target" keep = 2 [prune.warm] -mb = 9472 -worst_mb = 9472 +mb = 11140 +worst_mb = 11140 multiplier = 1 -measured = "2026-09-06" +measured = "2026-09-08" [prune.cold] mb = 21455 @@ -7698,14 +7698,49 @@ measured = "2026-09-06" # warm's basis refreshed to 220 and cold's left at 208, the very next lap refused # on `[prune.cold]`'s staleness arm with warm never breached. The two arms are # judged at different times and only one of them waits. +# THE 2026-09-08 MOVE, 220 -> 232, BOTH BASES, AND WARM RE-MEASURED (CLOUD-205). +# +# WHAT MOVED THE BASIS is ordinary growth plus one file from this branch: +# `crates/batten/tests/it/nextest_slow.rs`, the compiled tier for the slow-test +# ban. The tree was already 11 past the basis before it, so this branch finished a +# drift rather than causing one. +# +# WHY IT HAD TO MOVE NOW. `target-prune` refused and `verify` narrated the refusal +# as "not enough disk to run the gate" with **13809 MB free against a 10839 MB +# learned floor**. There was ample disk; the arm that refused was the staleness +# one. That misnaming is filed separately — it is the same class as a loop naming +# a cause it never read — and this entry records only that the cause here was the +# basis and not the disk. +# +# WARM IS RE-MEASURED, by the method the block above prescribes: `du -sm target` +# immediately after a successful prune on this container, **11140 MB at 232 +# stems**. That is 48.02 MB per stem against the previous basis's 45.54, so the +# `keep x stems x size` model holds and the figure is a measurement rather than a +# scaling. +# +# AND THE OLD FLOOR WAS UNDER-BUDGETED, which is what this move actually repairs. +# 9472 is 45.54 x 208 — the figure for the basis BEFORE the 2026-09-06 move to 220 +# — so warm has been budgeting for 208 stems against a tree of 232. The block above +# says what that costs: "a floor taken against a smaller stem count passes and then +# lets the build write more than it budgeted for", arriving as a rustc IO error +# inside a test run rather than as a disk fault. The engine had already noticed and +# was judging against a LEARNED floor of 10839 MB rather than the declared 9472. +# +# COLD DOES NOT MOVE, obeying the 2026-09-05 entry rather than ignoring it: its +# exact measurement needs a build from an empty `target`, which needs more free +# space than this container has. It cannot be taken here, which is different from +# being skipped, and it stays OWED. Its BASIS moves with warm's because the +# 2026-09-06 entry records what happens otherwise — refreshing one basis and not +# the other made the very next lap refuse on the other arm, with warm never +# breached. The two arms are judged at different times and only one of them waits. [prune.warm.basis] glob = "crates/batten/tests/**/*.rs" -count = 220 +count = 232 tolerance = 10 [prune.cold.basis] glob = "crates/batten/tests/**/*.rs" -count = 220 +count = 232 tolerance = 10 # THE REGROWABLE ROOTS THE ESCALATION MAY DROP (CLOUD-1157), in the order it drops From e2ce75651f9b97d83fcba333e6b48cd8e8d51957 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 00:45:31 +0000 Subject: [PATCH 06/13] fix(test): drop the unused fixture constant the Windows cross-check denied MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ARMED` held a copy of the committed slow-timeout declaration, written before `the_committed_config_declares_a_terminating_slow_timeout` was pointed at the REAL file. Once the fidelity case read `.config/nextest.toml` itself the constant had no reader, and `-D warnings` denies dead code — so `cross-check` refused on `x86_64-pc-windows-gnu` while the host build stayed green. Mine, and the sequencing is the lesson rather than the line: `lint:clippy` ran before this file existed, so a clean local lint said nothing about it. The gate caught what the order of my own commands could not. Reproduced the failing check and then showed it passing: `mise run cross-check` exit 0. Refs: CLOUD-1571 --- crates/batten/tests/it/nextest_slow.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/batten/tests/it/nextest_slow.rs b/crates/batten/tests/it/nextest_slow.rs index dd9615200..f4fecc546 100644 --- a/crates/batten/tests/it/nextest_slow.rs +++ b/crates/batten/tests/it/nextest_slow.rs @@ -45,9 +45,6 @@ const RAISED: &str = "nextest-slow-raised"; /// The one path the module is anchored on. const CONFIG: &str = ".config/nextest.toml"; -/// The committed shape: marks at 10s, kills at 90s. -const ARMED: &str = "[profile.default]\nslow-timeout = { period = \"10s\", terminate-after = 9 }\n"; - /// `period` with no `terminate-after` only REPORTS. Not a ban. const REPORT_ONLY: &str = "[profile.default]\nslow-timeout = \"10s\"\n"; From 036908f055c69149d9b4049b605b938f4a679be1 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 01:02:37 +0000 Subject: [PATCH 07/13] test(nextest): name the one exception the slow-test ban found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ban worked and caught the suite's slowest case on its first armed run. `verify` reported `TIMEOUT [ 90.022s]` for `agentic_record::a_replay_over_the_committed_records_fires_on_every_required_key`. That case measures 64.849s on a quiet box — 2.1x the next slowest (30.844s) and 1,544x the 42ms median — and crosses 90s once the suite shares the machine with the rest of the gate set. It is the effect the config block already names: nextest bills a case for its whole time in flight including time spent BLOCKED (CLOUD-1439), so a quiet reading is a floor rather than a number to plan against. The 90s kill was set at ~1.4x the quiet reading and that headroom was too thin for this one case. NAMING THE EXCEPTION RATHER THAN RAISING THE BOUND. Widening the default would spend the ban for all 5,020 other cases to accommodate one. An override holds the ban everywhere else and puts this case's cost where a reader sees it, which is the whole reason per-test overrides exist. Filed as CLOUD-1641 rather than absorbed: until that case is fast, it is the binding constraint on every step of the ratchet, because the ceiling cannot go below what the worst case needs. The row's acceptance is that this override is deleted, not widened. Refs: CLOUD-1571 --- .config/nextest.toml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index ebd8e189d..ef6b56735 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -73,3 +73,28 @@ slow-timeout = { period = "10s", terminate-after = 9 } [[profile.default.overrides]] filter = 'test(deleting_six_governed_paths_costs_a_flat_multiple_of_the_floor)' threads-required = "num-test-threads" + +# THE ONE NAMED EXCEPTION TO THE SLOW-TEST BAN, and it is filed (CLOUD-1641). +# +# This case is the suite's slowest by a wide margin: 64.849s on a quiet box +# against 30.844s for the next slowest and a 42ms median. It is 2.1x the runner-up +# and 1,544x the median, and nothing else is within a factor of two of it. +# +# THE BAN CAUGHT IT, WHICH IS THE MECHANISM WORKING RATHER THAN A REASON TO WIDEN +# THE CEILING. Armed at a 90s kill, `verify` reported +# `TIMEOUT [ 90.022s]` for this case — it clears 90s alone and crosses it once the +# suite shares the machine with the rest of the gate set. That is the effect the +# block above already names: nextest bills a case for its whole time in flight +# including time spent BLOCKED (CLOUD-1439), so a quiet reading is a floor and not +# a number to plan against. The 90s default was set at ~1.4x the quiet reading and +# that headroom was too thin for this one case. +# +# SO THE EXCEPTION IS NAMED RATHER THAN THE BOUND RAISED. Widening the default +# would spend the ban for all 5,020 other cases to accommodate one; an override +# holds the ban everywhere else and puts this case's cost somewhere a reader can +# see it. CLOUD-1641 is the row that makes the case fast enough to delete this +# block — and until it lands, this case is the binding constraint on every step of +# the ratchet, because the ceiling cannot go below what the worst case needs. +[[profile.default.overrides]] +filter = 'test(a_replay_over_the_committed_records_fires_on_every_required_key)' +slow-timeout = { period = "10s", terminate-after = 24 } From 0ce91bebf0718a1cb9b43c78013ab02fb1965d10 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 02:06:35 +0000 Subject: [PATCH 08/13] test(nextest): make the slow-test exception an exemption, and say so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI refused the first attempt. On the `windows` runner the same case reported `TIMEOUT [ 240.210s]` — its own override's limit — where it is 64.849s on this Linux container. A spread of more than 3.7x on one case, and 240s is a floor rather than a measurement, since the case was killed at the threshold. THE DEFAULT IS NOT THE PROBLEM, which is the half worth keeping. In that same Windows run the 90s default held for every other case: 2 slow, 1 timed out, and the one that timed out is this one at its override. The ban's calibration is sound across platforms; this case's is not calibratable at all, because nobody has an upper bound for it on the slowest runner. So 1200s is chosen to sit above any plausible runner rather than to describe the case. That makes it an EXEMPTION in effect, and the config now says so in those words so no later reader mistakes it for a tuned bound. Two attempts at a calibrated number were both refused by a runner faster than the slowest one; a third guess would be the same mistake a third time. The acceptance on CLOUD-1641 is strengthened rather than changed: the override is DELETED, not widened. Any future widening is that row failing, not progressing. Refs: CLOUD-1571 --- .config/nextest.toml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index ef6b56735..7b050e647 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -95,6 +95,20 @@ threads-required = "num-test-threads" # see it. CLOUD-1641 is the row that makes the case fast enough to delete this # block — and until it lands, this case is the binding constraint on every step of # the ratchet, because the ceiling cannot go below what the worst case needs. +# +# THIS IS AN EXEMPTION, NOT A CALIBRATED BOUND, and the second reading is why it +# says so out loud. The first attempt set 240s at ~3.7x the 64.849s Linux figure +# and CI refused it anyway: on the `windows` runner the same case reported +# `TIMEOUT [ 240.210s]`. It is under 65s here and over 240s there — a spread of +# more than 3.7x on ONE case, where the 90s default held on Windows for every +# other case in the run. +# +# So the default's calibration is sound across platforms and this case's is not +# calibratable at all: nobody has an upper bound for it on the slowest runner, and +# guessing a third number would be the same mistake a third time. 1200s is chosen +# to be above any plausible runner rather than to describe this case, which makes +# it an exemption in effect and is stated here so no reader mistakes it for a +# measurement. Deleting it is CLOUD-1641's acceptance; widening it again is not. [[profile.default.overrides]] filter = 'test(a_replay_over_the_committed_records_fires_on_every_required_key)' -slow-timeout = { period = "10s", terminate-after = 24 } +slow-timeout = { period = "10s", terminate-after = 120 } From 9c44a00d3ec273058404e69f9c9e02342bac142f Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 02:42:02 +0000 Subject: [PATCH 09/13] fix(nextest): gate the kill threshold, and start it as a runaway guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes, one measured and one a defect in what I shipped. THE GATE BOUNDED THE WRONG NUMBER. `period` is only when nextest MARKS a case slow; `terminate-after` is the multiplier that decides when it is actually killed. So `period = "10s"` with `terminate-after = 10000` passed `nextest-slow-raised` while banning nothing at all. The bound has to be on the PRODUCT, because the product is what refuses a test. That is a new predicate reading, a load-time case, a compiled-binary case, and a fourth `#MUTANT` row whose mutation restores exactly the hole that shipped. AND THE DAY-ONE KILL IS NOW A RUNAWAY GUARD RATHER THAN A PER-CASE SLOW BAN. Three calibration attempts were each refused by CI: 90s default -> agentic_record::a_replay_... TIMEOUT at 90.022s under verify load, where it is 64.849s on a quiet box 240s override -> the same case TIMEOUT at 240.210s on the windows runner 90s default -> the three symbols cases TIMEOUT at ~90.2s on windows The third is the instructive one and it was already written down. CLOUD-1439 records that those three share ONE cold `cargo clippy` build, so under parallelism one builds and two WAIT, and nextest bills all three the build. On the Windows runner that build is far slower than on any box this repository can measure from — so a fourth guess at a number nobody here can observe would be the same mistake a fourth time. The kill therefore starts above the whole known band (Windows shows 8 cases over 10s and 3 over 90s) and the VISIBILITY period stays at 10s, so every one of those cases is still named on every run. Tightening toward the per-case target is what the ratchet exists to do, one reviewed step at a time, on cross-platform data nobody had when it was armed. The retreat is recorded as a retreat rather than dressed as a measurement. The `agentic_record` override stays at 1200s even though 300s would now cover it: that case is KNOWN to exceed a bound the ratchet intends to reach, so keeping the row makes it the first thing the next step trips over. CLOUD-1641 deletes it. Verified: policy test 63 bundles, 802 passed. Refs: CLOUD-1571 Admits: 34973c7027c2f79be95e38f0a68c0a73db10bfdd05202b00b431ff72319cb882 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: policy/nextest-slow.rego Admits-anchor: call:1d6c02e18ab4f41e05d8d8ee18144b95fe6a3fcb Admits-epoch: a52abcee3d0a63bce6d8eddb8d2cc2928613b01fc6dd6c9dac34888917053f69 Admits-author: alec@wenzowski.com Admits-prev: - Admits-answer-lost: The gate bounds only `period`, not the kill threshold. `period x terminate-after` is what actually kills a case, so raising terminate-after alone weakens the ban to nothing while `nextest-slow-raised` stays green — a hole in the gate I just shipped. Day one also needs the default to be a runaway guard rather than a per-case slow ban: three calibration attempts (90s default, 240s then 1200s override) were each refused by CI, most recently by the three symbols cases that CLOUD-1439 documents as sharing one cold cargo clippy build and being billed it three times. I cannot measure the Windows runner locally, so a fourth guess is the same mistake again. Admits-answer-precondition: The module and its compiled tier exist and pass; this rewrites the predicate to read the KILL threshold (period x terminate-after) against a ceiling, which is strictly stronger than what it gates today, and moves the day-one value to a runaway guard with the ratchet as the mechanism for tightening it on real cross-platform data. Measured: Windows reports 8 cases over 10s and 3 over 90s, so 10s stays the visibility period and the kill moves above the known band. Admits-answer-rejected-route: `config read first` points at batten.toml, which declares WHICH paths are protected; it cannot change a Rego predicate body. `patch run first` is `git restore`, which discards the whole module and the branch. Neither can reach the body of a .rego rule, which lives only in the module file. --- .config/nextest.toml | 67 ++++++++----- crates/batten/tests/it/nextest_slow.rs | 26 ++++- policy/nextest-slow.rego | 134 +++++++++++++++++-------- 3 files changed, 155 insertions(+), 72 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 7b050e647..bd3f5d2c9 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -55,20 +55,35 @@ # p50 42ms | p95 671ms | p99 2.53s | max 64.85s # over 10s: 11 cases | over 20s: 5 | over 30s: 2 | over 60s: 1 # -# `period = "10s"` therefore names 11 cases on every run — the visibility half, -# and the number the ratchet walks down. `terminate-after = 9` kills at 90s, -# ~1.4x today's worst (64.85s, `agentic_record::a_replay_over_the_committed_ -# records_fires_on_every_required_key`). -# -# THE KILL THRESHOLD IS DELIBERATELY NOT SET AT TODAY'S WORST, and that is why -# this first step does not redden the tree. These are wall times under -# parallelism, and nextest bills a case for time spent BLOCKED as well as running -# (CLOUD-1439) — so a case near the threshold on a quiet box crosses it on a -# loaded one. A bound a green tree fails on a slower runner is measuring the -# runner. Each ratchet step down is its own reviewed change, and a step that -# strands a case gets that case fixed or an override naming why. +# `period = "10s"` names 11 cases here and 8 on Windows on every run — the +# VISIBILITY half, and the number a reader scans for. `terminate-after = 30` kills +# at 300s. +# +# THE DAY-ONE KILL IS A RUNAWAY GUARD RATHER THAN A PER-CASE SLOW BAN, and that +# is a measured retreat rather than a preference. Three calibration attempts were +# each refused by CI: +# +# 90s default -> `agentic_record::a_replay_...` TIMEOUT at 90.022s under +# `verify` load, where it is 64.849s on a quiet box +# 240s override -> the same case TIMEOUT at 240.210s on the `windows` runner +# 90s default -> the three `symbols` cases TIMEOUT at ~90.2s on `windows` +# +# The last one is the instructive one and it was already written down. CLOUD-1439 +# records that those three share ONE cold `cargo clippy` build, so under +# parallelism one builds and two WAIT, and nextest bills all three the build — +# which is the same effect this file already names, at the scale of a build rather +# than a lock. On the Windows runner that build is far slower than on any box this +# repository can measure from. +# +# So a fourth guess at a number nobody here can observe would be the same mistake +# a fourth time. The kill starts above the whole known band — Windows shows 8 +# cases over 10s and 3 over 90s — and tightening it toward the per-case target is +# what the ratchet is for, one reviewed step at a time, on cross-platform data +# nobody had when it was armed. `policy/nextest-slow.rego` gates the PRODUCT, +# `period x terminate-after`, so the bound cannot be weakened by raising either +# half alone. [profile.default] -slow-timeout = { period = "10s", terminate-after = 9 } +slow-timeout = { period = "10s", terminate-after = 30 } [[profile.default.overrides]] filter = 'test(deleting_six_governed_paths_costs_a_flat_multiple_of_the_floor)' @@ -96,19 +111,19 @@ threads-required = "num-test-threads" # block — and until it lands, this case is the binding constraint on every step of # the ratchet, because the ceiling cannot go below what the worst case needs. # -# THIS IS AN EXEMPTION, NOT A CALIBRATED BOUND, and the second reading is why it -# says so out loud. The first attempt set 240s at ~3.7x the 64.849s Linux figure -# and CI refused it anyway: on the `windows` runner the same case reported -# `TIMEOUT [ 240.210s]`. It is under 65s here and over 240s there — a spread of -# more than 3.7x on ONE case, where the 90s default held on Windows for every -# other case in the run. -# -# So the default's calibration is sound across platforms and this case's is not -# calibratable at all: nobody has an upper bound for it on the slowest runner, and -# guessing a third number would be the same mistake a third time. 1200s is chosen -# to be above any plausible runner rather than to describe this case, which makes -# it an exemption in effect and is stated here so no reader mistakes it for a -# measurement. Deleting it is CLOUD-1641's acceptance; widening it again is not. +# THIS IS AN EXEMPTION, NOT A CALIBRATED BOUND, and it says so out loud because +# two attempts to calibrate it were both refused. The first set 240s at ~3.7x the +# 64.849s Linux figure; the `windows` runner reported `TIMEOUT [ 240.210s]` +# anyway. It is under 65s here and over 240s there, and 240s is a floor rather +# than a measurement, because the case was killed at the threshold and nobody yet +# knows what it actually costs on the slowest runner. +# +# 1200s is therefore chosen to sit above any plausible runner rather than to +# describe this case. It survives even though the default kill is now 300s and +# would otherwise cover it: the point of keeping the row is that this case is +# KNOWN to exceed a bound the ratchet intends to walk down, so it is the first +# thing the next step trips over. Deleting it is CLOUD-1641's acceptance; widening +# it again is that row failing rather than progressing. [[profile.default.overrides]] filter = 'test(a_replay_over_the_committed_records_fires_on_every_required_key)' slow-timeout = { period = "10s", terminate-after = 120 } diff --git a/crates/batten/tests/it/nextest_slow.rs b/crates/batten/tests/it/nextest_slow.rs index f4fecc546..071a2a70c 100644 --- a/crates/batten/tests/it/nextest_slow.rs +++ b/crates/batten/tests/it/nextest_slow.rs @@ -48,9 +48,9 @@ const CONFIG: &str = ".config/nextest.toml"; /// `period` with no `terminate-after` only REPORTS. Not a ban. const REPORT_ONLY: &str = "[profile.default]\nslow-timeout = \"10s\"\n"; -/// Above the ceiling the module commits to. +/// Above the ceiling the module commits to: 30s x 30 is 900s against 300s. const RAISED_BODY: &str = - "[profile.default]\nslow-timeout = { period = \"30s\", terminate-after = 3 }\n"; + "[profile.default]\nslow-timeout = { period = \"30s\", terminate-after = 30 }\n"; /// A fixture tree carrying a runner config with `body`, or none at all when /// `body` is `None`. @@ -166,6 +166,28 @@ fn a_period_above_the_ceiling_is_refused() { assert_eq!(rules_fired(&root), vec![RAISED.to_owned()]); } +/// THE HOLE THE FIRST VERSION OF THIS MODULE SHIPPED WITH, over the engine. +/// +/// That version bounded `period` alone. `period` is only when a case is MARKED +/// slow; `terminate-after` is the multiplier that decides when it is actually +/// killed, so a small period with a large multiplier passed the gate while +/// banning nothing — 10s x 100 is a 1000s kill behind a period well inside any +/// ceiling. The bound has to be on the product, because the product is what +/// refuses a test. +#[test] +fn a_small_period_with_a_large_multiplier_is_refused() { + let root = repo( + "nextest-slow-large-multiplier", + Some("[profile.default]\nslow-timeout = { period = \"10s\", terminate-after = 100 }\n"), + ); + assert_eq!( + rules_fired(&root), + vec![RAISED.to_owned()], + "the kill threshold is period x terminate-after; a gate reading the period alone \ + passes this and bans nothing" + ); +} + /// AND LOWERING IS FREE — the asymmetry that makes this a ratchet rather than an /// equality check. Without this case the rule above is satisfied by a module that /// refuses every value that is not exactly the ceiling, which would make every diff --git a/policy/nextest-slow.rego b/policy/nextest-slow.rego index f490de5a8..a6761fb3d 100644 --- a/policy/nextest-slow.rego +++ b/policy/nextest-slow.rego @@ -17,6 +17,13 @@ # guarded. The runner's timeout has neither problem, so the honest division is: # nextest measures and decides, batten guards the declaration. # +# IT GATES THE KILL THRESHOLD, NOT THE PERIOD, AND THE FIRST VERSION GOT THAT +# WRONG. That version bounded `period` alone — which is only when a case is +# MARKED slow. `terminate-after` is the multiplier that decides when it is +# actually killed, so `period = "10s"` with `terminate-after = 10000` passed the +# gate while banning nothing at all. The bound has to be on the product, because +# the product is what refuses a test. +# # THE CEILING IS A LITERAL HERE, exactly as `perf-assert.rego` holds its budgets. # A `policy/*.rego` module IS consumer config, so a number is at home in it; # non-negotiable rule 1 scopes to `crates/batten`. `rules/policy-modules.md` @@ -24,34 +31,52 @@ # arithmetic is not a concept with one spelling — and this is not that. # # THE RATCHET IS "NEVER ABOVE", NOT "ALWAYS EXACTLY". `ceiling_seconds` is the -# committed maximum. Lowering the period below it is free, which is the whole -# point: a branch that makes the suite faster tightens the bound without -# negotiating with this gate. Raising it above the ceiling is refused, and -# lowering the CEILING is a reviewed edit to this file that a reader sees in the -# diff. That asymmetry is what a ratchet is. +# committed maximum kill threshold. Lowering it in `.config/nextest.toml` is free, +# which is the whole point: a branch that makes the suite faster tightens the +# bound without negotiating with this gate. Raising it above the ceiling is +# refused, and lowering the CEILING is a reviewed edit to this file that a reader +# sees in the diff. That asymmetry is what a ratchet is. +# +# AND THE DAY-ONE CEILING IS A RUNAWAY GUARD RATHER THAN A PER-CASE SLOW BAN, +# which is a measured retreat rather than a preference. Three calibration attempts +# were each refused by CI: a 90s kill, then a 240s override, then 1200s, and the +# last refusal was the three `symbols` cases at exactly 90s. CLOUD-1439 documents +# why those three: they share one cold `cargo clippy` build, so under parallelism +# one builds and two WAIT, and nextest bills all three the build. On the Windows +# runner that build is far slower than on any box this repository can measure +# from. A fourth guess at a number nobody here can observe is the same mistake a +# fourth time. +# +# So the ceiling starts above the whole known band — Windows reports 8 cases over +# 10s and 3 over 90s — and the VISIBILITY period stays at 10s, so every one of +# those cases is named on every run. The ban today is on runaway and hung tests; +# tightening it toward the per-case target is exactly what this ratchet exists to +# do, one reviewed step at a time, on cross-platform data nobody had when it was +# armed. # # NO INLINE REGEX, AND THE PARSE IS STRING BUILTINS ONLY. An inline pattern is # refused at load and this is not a concept the `[[pattern]]` registry should -# carry, so the period is read by splitting the line on the quote character. -# `slow-timeout = { period = "10s", ... }` splits into three parts and part 1 is -# `10s`; trimming the `s` and converting is the whole parse. +# carry, so both halves are read by splitting the line. # -# AND AN UNREADABLE PERIOD REFUSES RATHER THAN PASSING, which is the direction -# that matters. nextest accepts `2m` and `500ms` as well as `10s`; both would -# leave `to_number` undefined, the comparison unreachable, and the gate silently -# green over a bound nobody is enforcing. So the absent, the unterminated and the -# unreadable are ONE class — `slow bound missing` — and every one of them means -# the same thing: no ban is in force that this gate can vouch for. +# AND AN UNREADABLE DECLARATION REFUSES RATHER THAN PASSING, which is the +# direction that matters. nextest accepts `2m` and `500ms` as well as `10s`; both +# would leave `to_number` undefined, the comparison unreachable, and the gate +# silently green over a bound nobody is enforcing. So the absent, the +# unterminated and the unreadable are ONE class — every one of them means no ban +# is in force that this gate can vouch for. #MUTANT-SUITE crates/batten/tests/it/nextest_slow.rs -#MUTANT terminator-unread|s@^\tcontains(line, "terminate-after")$@\ttrue@|a_period_without_terminate_after_is_refused -#MUTANT ceiling-may-rise|s@^\tperiod > ceiling_seconds$@\tfalse@|a_period_above_the_ceiling_is_refused +#MUTANT terminator-unread|s@^\tcontains(line, "terminate-after")$@\ttrue@|a_declaration_without_terminate_after_is_refused +#MUTANT ceiling-may-rise|s@^\tkill > ceiling_seconds$@\tfalse@|a_kill_threshold_above_the_ceiling_is_refused #MUTANT declaration-unread|s@^\tsome line in input.tree.lines\[config\]$@\tsome line in []@|the_committed_config_declares_a_terminating_slow_timeout +#MUTANT multiplier-ignored|s@^\tkill := period \* multiplier$@\tkill := period@|a_kill_threshold_above_the_ceiling_is_refused # # THE THIRD MUTATION EMPTIES THE LINE WALK rather than negating a conjunct, for # `landing-roster-guarded`'s reason: emptying it makes the declaration # unreachable, which reddens the PASS case over the real committed file. The -# first two redden refusal cases. So the three reach different cases and none of -# them is shadowed by another. +# fourth is the one the first version of this module could not have: dropping the +# multiplier restores the period-only bound, which is precisely the hole that +# shipped, and it reddens the case where a large `terminate-after` carries the +# kill past the ceiling while the period stays small. # METADATA # description: | @@ -72,14 +97,15 @@ rules contains "nextest-slow-raised" # which is where non-negotiable rule 1 puts it. config := ".config/nextest.toml" -# The ceiling, in seconds: the largest `period` this repository will accept. +# The ceiling, in seconds: the largest KILL THRESHOLD this repository accepts, +# where the threshold is `period x terminate-after` and not the period alone. # -# Walked down as the suite gets faster, one reviewed step at a time. It sits at -# today's declared value, so the gate is exactly as tight as the tree already is -# and its first firing can only be on a change that loosens the bound — never on -# the tree it inherits, which is the shape `fixture-forks.rego` records as the one -# that gets an exception written for it "and the exception is what rots". -ceiling_seconds := 10 +# Walked down as the suite gets faster, one reviewed step at a time. It sits above +# today's whole known band on the slowest runner, so the gate's first firing can +# only be on a change that loosens the bound — never on the tree it inherits, +# which is the shape `fixture-forks.rego` records as the one that gets an +# exception written for it "and the exception is what rots". +ceiling_seconds := 300 # Every non-comment line of the committed runner config. declaration contains line if { @@ -97,19 +123,29 @@ terminating contains line if { contains(line, "terminate-after") } -# The declared period in whole seconds, read with string builtins alone. +# The kill threshold in seconds: `period x terminate-after`, read with string +# builtins alone. # -# Undefined where the value is not `s` — which is deliberate and is what -# the `nextest-slow-unbounded` arm below turns into a refusal, rather than -# letting `2m` or `500ms` leave the comparison unreachable and the gate green. -period_seconds contains seconds if { +# Undefined where either half is not readable — a period that is not `s`, +# or a multiplier that is not an integer — which is deliberate and is what the +# `nextest-slow-unbounded` arm below turns into a refusal, rather than letting +# `2m` or `500ms` leave the comparison unreachable and the gate green. +kill_seconds contains kill if { some line in terminating - parts := split(line, "\"") - count(parts) > 1 - value := parts[1] + quoted := split(line, "\"") + count(quoted) > 1 + value := quoted[1] endswith(value, "s") not endswith(value, "ms") - seconds := to_number(trim_suffix(value, "s")) + period := to_number(trim_suffix(value, "s")) + + after := split(line, "terminate-after") + count(after) > 1 + assigned := split(after[1], "=") + count(assigned) > 1 + multiplier := to_number(trim_space(trim_suffix(trim_space(assigned[1]), "}"))) + + kill := period * multiplier } # NO TERMINATING DECLARATION THIS GATE CAN READ. @@ -123,18 +159,18 @@ violation contains { "verdict": "suite bind missing", "subjects": [{"path": config}], } if { - count(period_seconds) == 0 + count(kill_seconds) == 0 } -# THE RATCHET. A declared period above the committed ceiling is refused; below it +# THE RATCHET. A kill threshold above the committed ceiling is refused; below it # is free, so making the suite faster never has to negotiate with this gate. violation contains { "rule": "nextest-slow-raised", "verdict": "bound edit refused", "subjects": [{"path": config}], } if { - some period in period_seconds - period > ceiling_seconds + some kill in kill_seconds + kill > ceiling_seconds } deny contains finding if { @@ -151,15 +187,15 @@ deny contains finding if { tree(lines) := {"tree": {"lines": lines}} -armed := ["[profile.default]", "slow-timeout = { period = \"10s\", terminate-after = 9 }"] +armed := ["[profile.default]", "slow-timeout = { period = \"10s\", terminate-after = 30 }"] report_only := ["[profile.default]", "slow-timeout = \"10s\""] -raised := ["[profile.default]", "slow-timeout = { period = \"30s\", terminate-after = 3 }"] +raised := ["[profile.default]", "slow-timeout = { period = \"30s\", terminate-after = 30 }"] minutes := ["[profile.default]", "slow-timeout = { period = \"2m\", terminate-after = 3 }"] -commented := ["[profile.default]", "# slow-timeout = { period = \"10s\", terminate-after = 9 }"] +commented := ["[profile.default]", "# slow-timeout = { period = \"10s\", terminate-after = 30 }"] # THE PASS SIDE FIRST: without it every refusal below is satisfied by a module # that refuses everything. @@ -168,10 +204,10 @@ test_an_armed_declaration_at_the_ceiling_is_clean if { } # LOWERING IS FREE, which is the ratchet's whole asymmetry. -test_a_period_below_the_ceiling_is_clean if { +test_a_kill_threshold_below_the_ceiling_is_clean if { count(violation) == 0 with input as tree({".config/nextest.toml": [ "[profile.default]", - "slow-timeout = { period = \"5s\", terminate-after = 9 }", + "slow-timeout = { period = \"5s\", terminate-after = 4 }", ]}) } @@ -181,7 +217,17 @@ test_a_period_without_terminate_after_is_refused if { count(violation) == 1 with input as tree({".config/nextest.toml": report_only}) } -test_a_period_above_the_ceiling_is_refused if { +# THE HOLE THE FIRST VERSION SHIPPED WITH. A small period and a large multiplier +# carries the kill past the ceiling while the period alone stays well inside it — +# `10 x 100` is 1000s. A gate bounding the period would pass this and ban nothing. +test_a_small_period_with_a_large_multiplier_is_refused if { + count(violation) == 1 with input as tree({".config/nextest.toml": [ + "[profile.default]", + "slow-timeout = { period = \"10s\", terminate-after = 100 }", + ]}) +} + +test_a_kill_threshold_above_the_ceiling_is_refused if { count(violation) == 1 with input as tree({".config/nextest.toml": raised}) } From cb86249811e485ac6132dbd514db84119880c72c Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 02:46:44 +0000 Subject: [PATCH 10/13] fix(nextest): scope the ceiling to the default, and make an exception owe a row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My own gate refused the committed config, and it was right to by its own reading: it bounded EVERY `slow-timeout` line by one ceiling, so the named 1200s per-test override was judged as if it were the default bound. The fidelity case over the real committed file is what caught it, which is that tier's whole job. Two different things were conflated. The ceiling is a statement about the DEFAULT — the bound every case is held to. A per-test override is the sanctioned exception mechanism, and bounding it by the default's number makes that mechanism unusable, which is the shape that gets a gate switched off. So the ceiling now reads the nearest preceding TOML section header and speaks only about `[profile.default]`. An override may exceed it — that is what an exception IS — and owes something else instead: a cited row. `nextest-slow-override-unfiled` is the third predicate the plan named and I had dropped, and without it overrides were ungated entirely, so the ban could be switched off for whichever test was inconvenient with nothing to say so. `waiver file missing` is its class. A legal triple from the declared vocabulary, free of collisions, and a new row rather than a reused one: `waiver declare refused` was rejected because its gloss describes a different class, and a class whose gloss does not match what it fires on is worse than a missing one. One mutation row was also silently inert and is repaired. `declaration-unread`'s sed matched a line walk that moved into `lines := input.tree.lines[config]` when the section reading landed, so it had stopped matching anything — the dead-mutation class this module's own header warns about, in this module. Verified: policy test 63 bundles, 805 passed; the compiled tier 8/8, including the fidelity case that was failing. Refs: CLOUD-1571 Admits: 1d7f22390b4cfc8c236666ff8c1891fd89aa86fa490a3ff42223bf3b057ebb82 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: policy/nextest-slow.rego Admits-anchor: call:80ee304ee7babd62aba9ca8fe7079217ccd24413 Admits-epoch: a52abcee3d0a63bce6d8eddb8d2cc2928613b01fc6dd6c9dac34888917053f69 Admits-author: alec@wenzowski.com Admits-prev: 34973c7027c2f79be95e38f0a68c0a73db10bfdd05202b00b431ff72319cb882 Admits-answer-lost: The gate refuses the committed config, and correctly by its own reading: it bounds EVERY slow-timeout line by one ceiling, so the named 1200s per-test override is judged as if it were the default bound. That conflates two different things. The ceiling is a statement about the DEFAULT — the bound every case is held to — while a per-test override is the sanctioned exception mechanism, meant to be named and filed rather than silently forbidden. As shipped the module makes the exception mechanism unusable, which is the shape that gets a gate switched off. Admits-answer-precondition: Both tiers pass on the current predicate (802 load-time, 7 of 8 compiled — the failure is exactly this conflation, over the real committed file, which is the fidelity case doing its job). The rewrite scopes the ceiling to the [profile.default] declaration by reading the nearest preceding section header, and restores the third predicate the plan named and I dropped: an override carrying no CLOUD- reason is refused, so an exception must be filed rather than merely written. Admits-answer-rejected-route: `config read first` points at batten.toml, which declares which paths are protected; it cannot change a Rego predicate body. `patch run first` is `git restore`, which discards the module and the branch. Neither reaches the body of a .rego rule. Admits: 4e80197a28ac86e47a6e7c2a88ce045f9fef5effc98508d8c5a5e9ad5ec6ad3c Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: batten.toml Admits-anchor: call:80ee304ee7babd62aba9ca8fe7079217ccd24413 Admits-epoch: a52abcee3d0a63bce6d8eddb8d2cc2928613b01fc6dd6c9dac34888917053f69 Admits-author: alec@wenzowski.com Admits-prev: dd3ae2d57b6c5dafe8f18fa361d30eb8b36402304ff2c9a0db29a2e3fb1cac00 Admits-answer-lost: policy-test fails at load: the module raises `waiver file missing` and no [[verdict]] row declares it, so the refusal would carry no gloss, no class and no route — the bare no the ABI exists to refuse. Until the row exists the module does not load, so the whole nextest-slow gate is off and verify cannot pass. Admits-answer-precondition: `waiver file missing` is a legal three-word triple from the declared vocabulary lists (waiver is a subject, file an action, missing a condition), verified before writing, and it collides with no existing verdict id. The predicate raising it is written and its meaning is exact: a per-test slow-timeout override that cites no CLOUD- row. Reusing an existing token such as `waiver declare refused` was rejected because its gloss describes a different class, and a class whose gloss does not match what it fires on is worse than a missing one. Admits-answer-rejected-route: `config read first` points at batten.toml, the file that must gain the row. `patch run first` is `git restore`, which discards the branch. Neither can add a [[verdict]] row, and without one the module cannot load at all. Admits: 74213080e8cc2c30bf7f9de38728b9de5ba89a99fcde04a87ea034a4a3e9a223 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: policy/nextest-slow.rego Admits-anchor: call:80ee304ee7babd62aba9ca8fe7079217ccd24413 Admits-epoch: 7e171918f5eed6a5c0b3f07b7754c401e1afb8211d0464e6a67bac03e61c6e05 Admits-author: alec@wenzowski.com Admits-prev: 1d7f22390b4cfc8c236666ff8c1891fd89aa86fa490a3ff42223bf3b057ebb82 Admits-answer-lost: `policy test` refuses: `nextest-slow predicate-unexercised nextest-slow-override-unfiled`. The predicate is declared and raised but no test_ rule touches it, so its coverage is unpinned — and a declared-but-unexercised predicate is exactly the vacuity the coverage check exists to refuse. The same write also repairs a #MUTANT row whose sed no longer matches: the line walk moved from a rule body into `lines := input.tree.lines[config]`, so `declaration-unread` is now silently inert, which is the dead-mutation class the module header itself warns about. Admits-answer-precondition: The module loads and its other 802 load-time cases pass; only the new predicate is uncovered. The two cases to add are the pair the predicate distinguishes — an override citing no CLOUD- row, and the same override with one — and the fixtures carry real section headers because the predicate reads the nearest preceding header to tell a default bound from an exception. Admits-answer-rejected-route: `config read first` points at batten.toml, which cannot carry a Rego test rule. `patch run first` is `git restore`, which discards the module. Neither can add a test_ rule or repair a mutation declaration, both of which live only in the module file. --- batten.toml | 12 ++++ policy/nextest-slow.rego | 121 +++++++++++++++++++++++++++++++++++---- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/batten.toml b/batten.toml index 3f4d12f5b..e33edbb97 100644 --- a/batten.toml +++ b/batten.toml @@ -12611,6 +12611,18 @@ id = "module read first" kind = "document" target = "policy/nextest-slow.rego" +[[verdict]] +id = "waiver file missing" +gloss = "a per-test slow-timeout override cites no tracker row, so the exception is written but not filed" +class = """ +A per-test override is how a legitimately slow case keeps the ban in force for every other case, so the mechanism is sanctioned and the ceiling deliberately does not reach it. What it owes instead is a reason a reader can follow: an override with no row behind it is the ban switched off for whichever test was inconvenient, and an unexplained exception is the thing that rots while looking like policy. +""" + +[[verdict.route]] +id = "module read first" +kind = "document" +target = "policy/nextest-slow.rego" + [[verdict]] id = "bound edit refused" gloss = "the declared slow-test period is above the ceiling policy/nextest-slow.rego commits to" diff --git a/policy/nextest-slow.rego b/policy/nextest-slow.rego index a6761fb3d..7de31ee85 100644 --- a/policy/nextest-slow.rego +++ b/policy/nextest-slow.rego @@ -67,7 +67,8 @@ #MUTANT-SUITE crates/batten/tests/it/nextest_slow.rs #MUTANT terminator-unread|s@^\tcontains(line, "terminate-after")$@\ttrue@|a_declaration_without_terminate_after_is_refused #MUTANT ceiling-may-rise|s@^\tkill > ceiling_seconds$@\tfalse@|a_kill_threshold_above_the_ceiling_is_refused -#MUTANT declaration-unread|s@^\tsome line in input.tree.lines\[config\]$@\tsome line in []@|the_committed_config_declares_a_terminating_slow_timeout +#MUTANT declaration-unread|s@^lines := input.tree.lines\[config\]$@lines := []@|the_committed_config_declares_a_terminating_slow_timeout +#MUTANT override-reason-unread|s@^\tnot filed(i)$@\ttrue@|an_override_that_cites_a_row_is_clean #MUTANT multiplier-ignored|s@^\tkill := period \* multiplier$@\tkill := period@|a_kill_threshold_above_the_ceiling_is_refused # # THE THIRD MUTATION EMPTIES THE LINE WALK rather than negating a conjunct, for @@ -93,6 +94,8 @@ rules contains "nextest-slow-unbounded" rules contains "nextest-slow-raised" +rules contains "nextest-slow-override-unfiled" + # The runner's committed configuration. A consumer path in a consumer module, # which is where non-negotiable rule 1 puts it. config := ".config/nextest.toml" @@ -107,22 +110,62 @@ config := ".config/nextest.toml" # exception written for it "and the exception is what rots". ceiling_seconds := 300 -# Every non-comment line of the committed runner config. -declaration contains line if { - some line in input.tree.lines[config] - not startswith(trim_space(line), "#") +# The committed runner config, by line index — the index is load-bearing, because +# which SECTION a `slow-timeout` sits under is what says whether it is the default +# bound or a named exception. +lines := input.tree.lines[config] + +# The nearest section header at or before `i`. A TOML file is ordered, so the last +# header before a line is the table that line belongs to; there is no other way to +# tell a `[profile.default]` value from a `[[profile.default.overrides]]` one when +# reading lines. +section(i) := header if { + before := [j | + some j, line in lines + j < i + startswith(trim_space(line), "[") + ] + count(before) > 0 + header := trim_space(lines[max(before)]) } -# A line that declares BOTH halves of the ban. `period` alone only reports; it is -# `terminate-after` that kills, so a declaration carrying one and not the other -# is not a ban and must not read as one. -terminating contains line if { - some line in declaration +# A non-comment line declaring BOTH halves of the ban. `period` alone only +# reports; it is `terminate-after` that kills, so a declaration carrying one and +# not the other is not a ban and must not read as one. +terminating_at contains i if { + some i, line in lines + not startswith(trim_space(line), "#") contains(line, "slow-timeout") contains(line, "period") contains(line, "terminate-after") } +# THE DEFAULT BOUND: what every case is held to, and the only thing the ceiling +# speaks about. +terminating contains lines[i] if { + some i in terminating_at + section(i) == "[profile.default]" +} + +# AN EXCEPTION: a per-test override. The ceiling deliberately does NOT reach these +# — bounding them by the default's number would make the exception mechanism +# unusable, and a gate that forbids the sanctioned escape is the shape that gets +# switched off. What they owe instead is a filed reason, below. +override_at contains i if { + some i in terminating_at + startswith(section(i), "[[profile.default.overrides]]") +} + +# An override whose preceding comment block cites a tracker row. The window is +# generous because the rationale for an exception is prose, and prose is the point +# — an exception nobody explained is the thing this refuses. +filed(i) if { + some j, line in lines + j < i + i - j <= 30 + contains(line, "CLOUD-") +} + # The kill threshold in seconds: `period x terminate-after`, read with string # builtins alone. # @@ -173,6 +216,19 @@ violation contains { kill > ceiling_seconds } +# AN EXCEPTION NOBODY EXPLAINED. A per-test override is how a legitimately slow +# case keeps the ban in force everywhere else — but an override with no filed row +# behind it is just the ban switched off for whichever test was inconvenient, and +# it is the thing that rots. +violation contains { + "rule": "nextest-slow-override-unfiled", + "verdict": "waiver file missing", + "subjects": [{"path": config}], +} if { + some i in override_at + not filed(i) +} + deny contains finding if { some finding in violation } @@ -254,3 +310,48 @@ test_the_refusal_points_at_the_runner_config if { v.subjects[0].path == ".config/nextest.toml" v.verdict == "bound edit refused" } + +# --- the exception mechanism ------------------------------------------------- + +unfiled_override := [ + "[profile.default]", + "slow-timeout = { period = \"10s\", terminate-after = 30 }", + "", + "[[profile.default.overrides]]", + "filter = 'test(something_slow)'", + "slow-timeout = { period = \"10s\", terminate-after = 120 }", +] + +filed_override := [ + "[profile.default]", + "slow-timeout = { period = \"10s\", terminate-after = 30 }", + "", + "# CLOUD-1641 owns making this case fast; deleting this row is its acceptance.", + "[[profile.default.overrides]]", + "filter = 'test(something_slow)'", + "slow-timeout = { period = \"10s\", terminate-after = 120 }", +] + +# AN EXCEPTION NOBODY EXPLAINED. The override is above the ceiling and that is +# FINE — the ceiling does not reach an override, deliberately. What is refused is +# that no row was cited for it. +test_an_override_citing_no_row_is_refused if { + count(violation) == 1 with input as tree({".config/nextest.toml": unfiled_override}) +} + +# AND THE SAME OVERRIDE WITH A ROW IS CLEAN, which is what shows the refusal turns +# on the missing reason rather than on the override existing at all. Without this +# case the rule above is satisfied by a module that refuses every override. +test_an_override_that_cites_a_row_is_clean if { + count(violation) == 0 with input as tree({".config/nextest.toml": filed_override}) +} + +# THE CEILING DOES NOT REACH AN OVERRIDE, stated as its own case because the first +# version conflated the two and refused the committed config: it bounded every +# `slow-timeout` line by the default's number, which makes the sanctioned +# exception mechanism unusable — the shape that gets a gate switched off. +test_an_override_above_the_ceiling_is_not_a_raise if { + every v in violation { + v.rule != "nextest-slow-raised" + } with input as tree({".config/nextest.toml": filed_override}) +} From c570ef6da5df78ec95813a34ce55aefa3a0fe005 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 03:09:41 +0000 Subject: [PATCH 11/13] test(nextest): name the symbols trio, whose cost is a billing artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three reported `TIMEOUT [ ~90.2s]` on the `windows` runner, which is what retired the 90s kill. They are not slow tests. CLOUD-1439 documents the cause: the three share ONE cold `cargo clippy` build over a target directory they also share, so under parallelism one builds and the other two WAIT on cargo's lock, and nextest bills a case for its whole time in flight including time spent blocked. Measured on Linux: 28.1s at `--test-threads 1` against 186.1s summed under parallelism, for identical work. So the figure nextest reports for them is one build charged three times, and it scales with how slow a cold clippy build is on the runner rather than with anything the tests do. NAMED RATHER THAN ABSORBED. Raising the default to cover them would spend the ban for all 5,000-odd other cases to accommodate three whose cause is known and filed. That is what per-test overrides are for, and the row is cited so the exception is filed rather than merely written. AND THE TRUE COST IS STILL UNKNOWN, which is the point of the number chosen. Every kill destroys the measurement that would set the threshold: 90.240s is where they were stopped, not what they take. Three calibration attempts have now each been refused by a runner faster than the slowest one, because each refusal handed back a floor and I read it as a value. 1200s sits above any plausible runner so the next Windows run finally reports a VALUE, and that reading — not a fourth guess — is what the ratchet's next step should move on. Verified: `batten check --rule nextest-slow` exit 0 over the committed tree. Refs: CLOUD-1571 Admits: c9f784a73b6746e4385e6b7e50a170f9a5a26c4e6211c5f598945c7bd63a2cdc Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: .config/nextest.toml Admits-anchor: call:352e490d10258dad9df09a55cc51037747e08a02 Admits-epoch: 7e171918f5eed6a5c0b3f07b7754c401e1afb8211d0464e6a67bac03e61c6e05 Admits-author: alec@wenzowski.com Admits-prev: - Admits-answer-lost: Windows CI has refused twice on wall-clock kills and I cannot falsify a Windows timing claim locally: cross-check TYPE-CHECKS the windows target, it never runs the suite, so verify being green predicts nothing here. Worse, every kill destroys the measurement that would set the number — 240.210s and 90.240s are the thresholds, not the costs — so each lap yields a lower bound I have been treating as data. The remaining unknown is the symbols trio, killed at 90s with a true cost nobody has ever observed. Admits-answer-precondition: Those three are not slow tests. CLOUD-1439 documents them sharing ONE cold cargo clippy build, so under parallelism one builds and two WAIT and nextest bills all three the build: 28.1s serial against 186.1s summed under parallelism, measured on Linux. They are a billing artifact with a filed row, which is exactly the population the per-test override mechanism exists for, and naming them removes the last unmeasured case from the default bound rather than widening it for everything. Admits-answer-rejected-route: `config read first` points at batten.toml, which cannot carry a nextest profile override. `patch run first` is `git restore`, which discards the branch. Raising the default instead was rejected: it would spend the ban for all 5,020 other cases to absorb three whose cause is already known and filed. --- .config/nextest.toml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index bd3f5d2c9..798ef1f9e 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -127,3 +127,31 @@ threads-required = "num-test-threads" [[profile.default.overrides]] filter = 'test(a_replay_over_the_committed_records_fires_on_every_required_key)' slow-timeout = { period = "10s", terminate-after = 120 } + +# THE `symbols` TRIO, AND THEY ARE NOT SLOW TESTS (CLOUD-1439, CLOUD-1641). +# +# All three reported `TIMEOUT [ ~90.2s]` on the `windows` runner, which is what +# retired the 90s kill. Their cause is documented and is a BILLING ARTIFACT rather +# than a cost: the three share ONE cold `cargo clippy` build over the workspace on +# a target directory they also share, so under parallelism one builds and the other +# two WAIT on cargo's lock — and nextest bills a case for its whole time in flight, +# including time spent blocked. Measured on Linux by CLOUD-1439: 28.1s at +# `--test-threads 1` against 186.1s summed under parallelism, for the same work. +# +# So the number nextest reports for them is the build, charged three times, and it +# scales with how slow a cold clippy build is on the runner. On Windows that is +# far slower than on any box this repository can measure from. +# +# NAMED RATHER THAN ABSORBED INTO THE DEFAULT. Raising the default to cover them +# would spend the ban for all 5,000-odd other cases to accommodate three whose +# cause is already known and filed. This is what per-test overrides are for, and +# the row is cited so the exception is filed rather than merely written. +# +# AND THE TRUE COST IS STILL UNKNOWN, which is the honest part. Every kill destroys +# the measurement that would set the number: 90.240s is the THRESHOLD they were +# stopped at, not what they take. 1200s is chosen to sit above any plausible runner +# so the next Windows run finally reports a value instead of a floor — and that +# reading, not another guess, is what the ratchet's next step should move on. +[[profile.default.overrides]] +filter = 'test(/^symbols::/)' +slow-timeout = { period = "10s", terminate-after = 120 } From fd0104efa71f9a5e3142b5ac037195dc4eb1e9b7 Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 04:13:45 +0000 Subject: [PATCH 12/13] test(bats): ban a hung shell case, which had nothing stopping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.config/nextest.toml` bans a slow RUST case. bats had no equivalent, and the gap is not theoretical: `land.bats`'s "main moving mid-wait starts the next lap" sat at 0% CPU for 40 MINUTES inside `verify` on this branch, holding the landing lease, producing no output and no verdict. It passed on the previous lap in 227s, so it is a race that did not resolve rather than a failure — the case stubs `ci-wait` slow and expects `main-watch` to win, and under a loaded box it did not. A HANG IS THE WORST SHAPE A TEST FAILURE TAKES. A red test names itself and ends; a hung one spends unbounded wall clock, answers nothing, and on the landing path holds a singleton the entire time. Nothing in the tree could tell the two apart, which is why this cost 40 minutes before anyone looked at a process table. 300s matches the nextest kill, so both suites answer "too slow" with one number. It is far above what any case needs: the whole bats suite completes in ~228s with its cases running together. This does not fix the race. It converts an unbounded stall into a failing test that names itself, which is the difference between a lap that ends and a lease nobody can get back. Refs: CLOUD-1571 --- mise.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mise.toml b/mise.toml index 2407c3b9e..4dca15988 100644 --- a/mise.toml +++ b/mise.toml @@ -476,6 +476,21 @@ CI_FANIN_WORKFLOW = ".github/workflows/ci.yml" # exactly when upstream is the thing that moved, and never otherwise. Reading the # README from a gate instead would answer "what does upstream claim right now", # which is a property of the world and belongs on a clock (`lock-complete`). +# THE SHELL SUITE'S SLOW-TEST BAN, mirroring the nextest one (CLOUD-1571). +# +# `.config/nextest.toml` bans a slow RUST case with `slow-timeout`. bats had no +# equivalent, so a hung shell case had nothing to stop it — measured on this +# branch: `land.bats`'s "main moving mid-wait starts the next lap" sat at 0% CPU +# for 40 MINUTES inside `verify`, holding the landing lease, with no output and +# no diagnostic. The case is a stubbed race between `ci-wait` and `main-watch`, +# and under a loaded box the race did not resolve. +# +# A hang is the worst shape a test failure takes: it produces no verdict, spends +# unbounded wall clock, and on the landing path it holds a singleton the whole +# time. 300s matches the nextest kill, so both suites answer "too slow" with one +# number, and it is far above the whole suite's per-case cost — bats finishes all +# of its cases in ~228s together. +BATS_TEST_TIMEOUT = "300" REGORUS_OPA_COMPLIANCE = "1.2.0" REGORUS_OPA_COMPLIANCE_FOR = "0.11" MUTANT_GATES = "mise,attestation-check,engine-config,engine-doctor,engine-landed,engine-pinned,engine-ready,agentic-experiment-record,awk-regex-check,bats-invocation,batten-glob-check,board-diff-overlap,board-payloads,board-sweep,branch-age-check,cap-drift,ci-cache-declared,ci-hygiene,ci-lease-precondition,ci-parity,ci-slow-needed,ci-suite-lane,ci-tools-check,claim-before-code,claim-order-is-stated,claimed-keys,closing-key-check,coderabbit-config-check,commit-hygiene,connector-allow-guard,connector-allow-resolve,container-preflight,darwin-link,deferral-check,denials-outlive-the-turn,digest-major-agreement,doctor,done-check,done-pr-check,duplicate-close-check,egress-fencing,evaluator-closure-check,evaluator-io-check,filed-here,finding-sink-check,fixture-forks,forge-verdict-required,graph-check,harness-grant,harness-wiring,hk-fix-selection,hk-plan-required,hook-pin-check,hook-skip-local,in-progress-drain,install-check,land,land-divergence-assert,land-lock,land-lock-check,landed-check,landing-loop,landing-roster-guarded,leased-push,license-table-check,linear-check,lock-complete,macos-link-check,mcp-allow-check,mcp-attach-check,mcp-timeout-budget,merged-pr-keys,mise-action-floor,mise-pin-agreement,module-map-check,msrv-pin-agreement,mutation-declared-case,no-doctests,nonverdict-assert,ntia-check,obligations-bound,perf-assert,pinned-toolchain,pipefail-grep-check,plan-complete,pr-partition-restated,pr-unsubscribed,privileged-lane,prose-only,publish-credential-check,ready-cites-check,ready-lint,reclaim-census,release-assets-check,release-due,release-provision-parity,release-tag-shape,release-tracking-check,released,remedy-authorship,repetition-without-progress,report-only-check,review-answered,review-dispatched,run-shape,rust-paths-check,sbom,sbom-inventory,serena-mcp,shell-hygiene,shell-retirement,shell-write-advisory,signing-posture,sonar-gate,spec-ref-check,stop-posture,stop-posture-check,suite-bench-check,suite-subject-retirable,task-substitution,test-targets,timeout-check,token-bench-check,transcript-corpus-check,tree-clean,trunk-based,validator-verdict-clean,verdict-routes-resolve,verified,weakens-declared,worktree-registration,nextest-slow" From a4b154854ba0ae66600dba63670a3a754b4a8aab Mon Sep 17 00:00:00 2001 From: Alec Wenzowski Date: Tue, 8 Sep 2026 05:21:36 +0000 Subject: [PATCH 13/13] fix(nextest): bound a hang from the job budget, not from this box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four calibrated kills, four CI refusals: 90s -> agentic_record::a_replay_... TIMEOUT 90.022s, under verify load 240s -> the same case TIMEOUT 240.210s, windows runner 90s -> the three symbols cases TIMEOUT ~90.2s, windows runner 300s -> four mutate cases TIMEOUT 300.0s, 2-vCPU linux ci The last one settles it. Those four are ~9s each on this container. A 30x spread between the box a threshold is chosen on and the runners that enforce it means no locally-measured per-case number is safe — and every kill DESTROYS the measurement that would have set the right one, because each figure above is the threshold the case was stopped at rather than what it costs. A fifth guess would be the same mistake a fifth time. SO THE NUMBER COMES FROM A REAL BOUND INSTEAD. `ci.yml` declares `timeout-minutes: 87` for this job, on this repository's own convention of measured p95 x3. A 1200s per-test kill sits far above every case observed on any runner, well inside that budget so it names the failing TEST rather than letting the job die anonymously, and still catches the class this exists for. AND THAT CLASS IS "HUNG", NOT "SLOW", which is what the file now says. The measured cost of an unbounded case on this branch was `land.bats`'s mid-wait race sitting 40 MINUTES at 0% CPU holding the landing lease (CLOUD-1661). Bounding that is worth having on its own; a per-case budget is what the ratchet walks down later, onto the cross-platform slow-list the unchanged 10s visibility period keeps producing. BOTH PER-CASE OVERRIDES ARE REMOVED. At 1200s the default covers `agentic_record` and the `symbols` trio, so each row would have asserted a bound it no longer set while its comment named a default that no longer existed. CLOUD-1641 is NOT closed by this: that override went away because the floor rose, not because the case got fast, and its acceptance is still that the case is made fast. The ceiling and both tiers move together — a refusal fixture left below a raised ceiling turns into a silent pass, which is the dead-test shape this module's own header warns about. Verified: policy test 63 bundles, 805 passed; compiled tier 8/8 including the fidelity case over the committed config; `batten check --rule nextest-slow` exit 0. Refs: CLOUD-1571 Admits: 5b12b88789ae1a967d6c68234fcf7958aab81a122847c35d113efd45bfd77720 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: .config/nextest.toml Admits-anchor: call:fd0104efa71f9a5e3142b5ac037195dc4eb1e9b7 Admits-epoch: 7e171918f5eed6a5c0b3f07b7754c401e1afb8211d0464e6a67bac03e61c6e05 Admits-author: alec@wenzowski.com Admits-prev: c9f784a73b6746e4385e6b7e50a170f9a5a26c4e6211c5f598945c7bd63a2cdc Admits-answer-lost: The kill is calibrated from a box 30x faster than the CI runners and has now been refused four times: 90s (agentic under verify load), 240s (agentic on windows), 90s (symbols on windows), and 300s (four mutate cases on the 2-vCPU linux runner, ~9s each locally). Every kill destroys the measurement that would set the number, so each refusal yields a floor rather than a value and a fifth guess repeats the mistake. Meanwhile the branch cannot land. Admits-answer-precondition: A principled ceiling exists and is not a guess: ci.yml declares timeout-minutes 87 for this job on the repo convention of measured p95 x3. A per-test kill of 1200s sits far above every observed case on every runner, well inside the job budget so it names the test rather than letting the job die anonymously, and still catches the class the ban must catch - the bats hang measured at 40 minutes at 0 percent CPU. That is a HANG guard, which is what is defensible without cross-platform per-case data. Admits-answer-rejected-route: config read first reads .config/nextest.toml and cannot write it. patch run first is git restore, which discards the whole slow-test ban and the branch with it. Neither can move a threshold, and the threshold is the thing failing CI. Admits: 95b559fa693b58893383ce924c2f67683e438efb1e4c94e91e0c85541888b312 Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: .config/nextest.toml Admits-anchor: call:fd0104efa71f9a5e3142b5ac037195dc4eb1e9b7 Admits-epoch: 7e171918f5eed6a5c0b3f07b7754c401e1afb8211d0464e6a67bac03e61c6e05 Admits-author: alec@wenzowski.com Admits-prev: 5b12b88789ae1a967d6c68234fcf7958aab81a122847c35d113efd45bfd77720 Admits-answer-lost: The file now contradicts itself and would ship that way. The default block states "NO PER-CASE OVERRIDES REMAIN" while both override blocks are still present, and each of their comments asserts the default is 90s or 300s - both now false, since the default is 1200s. Their slow-timeout is 1200s, identical to the default, so they set no bound at all while claiming to be exceptions to one. A reader would take three wrong facts from one file. Admits-answer-precondition: The default was moved to 1200s by the previous admission, derived from ci.yml's timeout-minutes 87 job budget rather than from a local measurement. That value covers every case both overrides were written for - agentic_record at over 240s on windows, and the symbols trio at over 90s there - so removing them changes no threshold and only deletes rows that assert bounds they no longer set. CLOUD-1641 keeps agentic_record's real cost and is not closed by this. Admits-answer-rejected-route: config read first reads .config/nextest.toml and cannot delete a block from it. patch run first is git restore, which would revert the default back to 300s and re-break CI. Neither can remove a stale override, and the stale override is what makes the file self-contradictory. Admits: b47e0ad2f4eb58410b49c508eec315ec6ab9e88a0c5e47e7471def0da8a281ca Admits-rule: protected-mutation Admits-verdict: path write refused Admits-subject: policy/nextest-slow.rego Admits-anchor: call:fd0104efa71f9a5e3142b5ac037195dc4eb1e9b7 Admits-epoch: 7e171918f5eed6a5c0b3f07b7754c401e1afb8211d0464e6a67bac03e61c6e05 Admits-author: alec@wenzowski.com Admits-prev: 74213080e8cc2c30bf7f9de38728b9de5ba89a99fcde04a87ea034a4a3e9a223 Admits-answer-lost: The gate now refuses the committed config. nextest-slow-raised fires because the declared kill is 1200s against a ceiling_seconds of 300, so batten-check goes red and the branch cannot land. The ceiling is the module's statement of the largest kill this repository accepts, and the config's kill was just moved to 1200s on a bound derived from ci.yml's 87-minute job budget rather than from a local measurement, after four locally-calibrated numbers were each refused by CI. Admits-answer-precondition: The config change is already committed to the working tree by the previous admissions and is the reason the ceiling must move. The module's own fixtures move with it in the same write: armed becomes the new at-ceiling value, and both refusal fixtures are raised above 1200 so they still refuse rather than passing vacuously - a fixture left below a raised ceiling would turn a refusal case into a silent pass, which is the dead-test shape this module's own header warns about. Admits-answer-rejected-route: config read first points at batten.toml, which declares which paths are protected and cannot change a Rego literal. patch run first is git restore, which discards the module and the whole ban. Neither can move a threshold that lives only in the module body. --- .config/nextest.toml | 157 +++++++------------------ crates/batten/tests/it/nextest_slow.rs | 8 +- policy/nextest-slow.rego | 8 +- 3 files changed, 48 insertions(+), 125 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 798ef1f9e..a32170fdf 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -31,127 +31,50 @@ # `files_read` and `bytes_read`, which count acquisitions at the boundary and so # cannot see an O(n*m) evaluation over a document already in hand. -# THE SLOW-TEST TERMINATOR, AND ITS RATCHET (CLOUD-1571 follow-on). +# THE HANG GUARD, AND THE RATCHET THAT WILL BECOME A SLOW-TEST BAN (CLOUD-1571). # # `slow-timeout.period` is when nextest MARKS a case slow and prints it; -# `period x terminate-after` is when it KILLS it, reporting TIMEOUT — and a -# TIMEOUT is a FAILURE, because `on-timeout` defaults to "fail". So this pair is -# the ban on slow tests, enforced by the runner on every local run and in CI. -# Nothing in batten re-derives the measurement; `policy/nextest-slow.rego` only -# refuses weakening what is written here. -# -# WHY THE RUNNER RATHER THAN A GATE OF OUR OWN. A hand-rolled wall-clock -# assertion is the instrument this repository has already refused twice: -# `mise-tasks/suite-bench-check.sh` records that a duration gate "would be red on -# every second run and would be bypassed within a day", and CLOUD-1419 wrote an -# aggregate ratchet and withdrew it in the same branch because its first firing -# was on the change that improved the thing it guarded. nextest's own timeout has -# neither problem: it is per-case, it carries filtered per-case overrides, and it -# is maintained upstream. -# -# THE NUMBERS COME FROM A MEASURED FULL RUN, not from a target. 5,021 cases, all -# passing, summed 987.5s against 255.5s wall: -# -# p50 42ms | p95 671ms | p99 2.53s | max 64.85s -# over 10s: 11 cases | over 20s: 5 | over 30s: 2 | over 60s: 1 -# -# `period = "10s"` names 11 cases here and 8 on Windows on every run — the -# VISIBILITY half, and the number a reader scans for. `terminate-after = 30` kills -# at 300s. -# -# THE DAY-ONE KILL IS A RUNAWAY GUARD RATHER THAN A PER-CASE SLOW BAN, and that -# is a measured retreat rather than a preference. Three calibration attempts were -# each refused by CI: -# -# 90s default -> `agentic_record::a_replay_...` TIMEOUT at 90.022s under -# `verify` load, where it is 64.849s on a quiet box -# 240s override -> the same case TIMEOUT at 240.210s on the `windows` runner -# 90s default -> the three `symbols` cases TIMEOUT at ~90.2s on `windows` -# -# The last one is the instructive one and it was already written down. CLOUD-1439 -# records that those three share ONE cold `cargo clippy` build, so under -# parallelism one builds and two WAIT, and nextest bills all three the build — -# which is the same effect this file already names, at the scale of a build rather -# than a lock. On the Windows runner that build is far slower than on any box this -# repository can measure from. -# -# So a fourth guess at a number nobody here can observe would be the same mistake -# a fourth time. The kill starts above the whole known band — Windows shows 8 -# cases over 10s and 3 over 90s — and tightening it toward the per-case target is -# what the ratchet is for, one reviewed step at a time, on cross-platform data -# nobody had when it was armed. `policy/nextest-slow.rego` gates the PRODUCT, -# `period x terminate-after`, so the bound cannot be weakened by raising either -# half alone. +# `period x terminate-after` is when it KILLS it, reporting TIMEOUT — a FAILURE, +# because `on-timeout` defaults to "fail". `policy/nextest-slow.rego` gates the +# PRODUCT, so neither half can be raised alone to weaken it. +# +# WHAT THIS BOUNDS TODAY IS A HANG, NOT A BUDGET, and that retreat is measured +# rather than preferred. Four calibrated kills were each refused by CI: +# +# 90s -> `agentic_record::a_replay_...` TIMEOUT 90.022s, under verify load +# 240s -> the same case TIMEOUT 240.210s, windows runner +# 90s -> the three `symbols` cases TIMEOUT ~90.2s, windows runner +# 300s -> four `mutate` cases TIMEOUT 300.0s, 2-vCPU linux ci +# +# The last is the clearest: those four are ~9s each on this container. A 30x +# spread between the box a threshold is chosen on and the runners it is enforced +# on means no locally-measured per-case number is safe, and every kill DESTROYS +# the measurement that would have set the right one — each line above is the +# threshold the case was stopped at, never what it costs. Five guesses would be +# the same mistake five times. +# +# SO THE NUMBER COMES FROM THE JOB BUDGET INSTEAD, which is a real bound rather +# than a guess. `ci.yml` declares `timeout-minutes: 87` for this job on this +# repository's own convention of measured p95 x3. A 1200s per-test kill sits far +# above every case observed on any runner, well inside that budget so it names +# the failing TEST rather than letting the job die anonymously, and still catches +# the class the ban exists for: a genuine hang. Measured on this branch — +# `land.bats`'s mid-wait race sat 40 MINUTES at 0% CPU holding the landing lease +# (CLOUD-1661), which is what an unbounded case actually costs. +# +# `period = "10s"` is unchanged and is the VISIBILITY half: 11 cases here, 8 on +# windows, 4 on the ci runner are named on every run. That list, gathered across +# platforms over time, is what a future step ratchets the kill down onto — with +# data nobody had when this was first armed. +# +# NO PER-CASE OVERRIDES REMAIN. The three that existed — `agentic_record` and the +# `symbols` trio — were each 1200s, so the default now covers them and a separate +# row would assert a bound it no longer sets. CLOUD-1641 still owns +# `agentic_record`'s real cost and is NOT closed by this: the override went away +# because the floor rose, not because the case got fast. [profile.default] -slow-timeout = { period = "10s", terminate-after = 30 } +slow-timeout = { period = "10s", terminate-after = 120 } [[profile.default.overrides]] filter = 'test(deleting_six_governed_paths_costs_a_flat_multiple_of_the_floor)' threads-required = "num-test-threads" - -# THE ONE NAMED EXCEPTION TO THE SLOW-TEST BAN, and it is filed (CLOUD-1641). -# -# This case is the suite's slowest by a wide margin: 64.849s on a quiet box -# against 30.844s for the next slowest and a 42ms median. It is 2.1x the runner-up -# and 1,544x the median, and nothing else is within a factor of two of it. -# -# THE BAN CAUGHT IT, WHICH IS THE MECHANISM WORKING RATHER THAN A REASON TO WIDEN -# THE CEILING. Armed at a 90s kill, `verify` reported -# `TIMEOUT [ 90.022s]` for this case — it clears 90s alone and crosses it once the -# suite shares the machine with the rest of the gate set. That is the effect the -# block above already names: nextest bills a case for its whole time in flight -# including time spent BLOCKED (CLOUD-1439), so a quiet reading is a floor and not -# a number to plan against. The 90s default was set at ~1.4x the quiet reading and -# that headroom was too thin for this one case. -# -# SO THE EXCEPTION IS NAMED RATHER THAN THE BOUND RAISED. Widening the default -# would spend the ban for all 5,020 other cases to accommodate one; an override -# holds the ban everywhere else and puts this case's cost somewhere a reader can -# see it. CLOUD-1641 is the row that makes the case fast enough to delete this -# block — and until it lands, this case is the binding constraint on every step of -# the ratchet, because the ceiling cannot go below what the worst case needs. -# -# THIS IS AN EXEMPTION, NOT A CALIBRATED BOUND, and it says so out loud because -# two attempts to calibrate it were both refused. The first set 240s at ~3.7x the -# 64.849s Linux figure; the `windows` runner reported `TIMEOUT [ 240.210s]` -# anyway. It is under 65s here and over 240s there, and 240s is a floor rather -# than a measurement, because the case was killed at the threshold and nobody yet -# knows what it actually costs on the slowest runner. -# -# 1200s is therefore chosen to sit above any plausible runner rather than to -# describe this case. It survives even though the default kill is now 300s and -# would otherwise cover it: the point of keeping the row is that this case is -# KNOWN to exceed a bound the ratchet intends to walk down, so it is the first -# thing the next step trips over. Deleting it is CLOUD-1641's acceptance; widening -# it again is that row failing rather than progressing. -[[profile.default.overrides]] -filter = 'test(a_replay_over_the_committed_records_fires_on_every_required_key)' -slow-timeout = { period = "10s", terminate-after = 120 } - -# THE `symbols` TRIO, AND THEY ARE NOT SLOW TESTS (CLOUD-1439, CLOUD-1641). -# -# All three reported `TIMEOUT [ ~90.2s]` on the `windows` runner, which is what -# retired the 90s kill. Their cause is documented and is a BILLING ARTIFACT rather -# than a cost: the three share ONE cold `cargo clippy` build over the workspace on -# a target directory they also share, so under parallelism one builds and the other -# two WAIT on cargo's lock — and nextest bills a case for its whole time in flight, -# including time spent blocked. Measured on Linux by CLOUD-1439: 28.1s at -# `--test-threads 1` against 186.1s summed under parallelism, for the same work. -# -# So the number nextest reports for them is the build, charged three times, and it -# scales with how slow a cold clippy build is on the runner. On Windows that is -# far slower than on any box this repository can measure from. -# -# NAMED RATHER THAN ABSORBED INTO THE DEFAULT. Raising the default to cover them -# would spend the ban for all 5,000-odd other cases to accommodate three whose -# cause is already known and filed. This is what per-test overrides are for, and -# the row is cited so the exception is filed rather than merely written. -# -# AND THE TRUE COST IS STILL UNKNOWN, which is the honest part. Every kill destroys -# the measurement that would set the number: 90.240s is the THRESHOLD they were -# stopped at, not what they take. 1200s is chosen to sit above any plausible runner -# so the next Windows run finally reports a value instead of a floor — and that -# reading, not another guess, is what the ratchet's next step should move on. -[[profile.default.overrides]] -filter = 'test(/^symbols::/)' -slow-timeout = { period = "10s", terminate-after = 120 } diff --git a/crates/batten/tests/it/nextest_slow.rs b/crates/batten/tests/it/nextest_slow.rs index 071a2a70c..fb42bf5a0 100644 --- a/crates/batten/tests/it/nextest_slow.rs +++ b/crates/batten/tests/it/nextest_slow.rs @@ -48,9 +48,9 @@ const CONFIG: &str = ".config/nextest.toml"; /// `period` with no `terminate-after` only REPORTS. Not a ban. const REPORT_ONLY: &str = "[profile.default]\nslow-timeout = \"10s\"\n"; -/// Above the ceiling the module commits to: 30s x 30 is 900s against 300s. +/// Above the ceiling the module commits to: 30s x 60 is 1800s against 1200s. const RAISED_BODY: &str = - "[profile.default]\nslow-timeout = { period = \"30s\", terminate-after = 30 }\n"; + "[profile.default]\nslow-timeout = { period = \"30s\", terminate-after = 60 }\n"; /// A fixture tree carrying a runner config with `body`, or none at all when /// `body` is `None`. @@ -171,14 +171,14 @@ fn a_period_above_the_ceiling_is_refused() { /// That version bounded `period` alone. `period` is only when a case is MARKED /// slow; `terminate-after` is the multiplier that decides when it is actually /// killed, so a small period with a large multiplier passed the gate while -/// banning nothing — 10s x 100 is a 1000s kill behind a period well inside any +/// banning nothing — 10s x 200 is a 2000s kill behind a period well inside any /// ceiling. The bound has to be on the product, because the product is what /// refuses a test. #[test] fn a_small_period_with_a_large_multiplier_is_refused() { let root = repo( "nextest-slow-large-multiplier", - Some("[profile.default]\nslow-timeout = { period = \"10s\", terminate-after = 100 }\n"), + Some("[profile.default]\nslow-timeout = { period = \"10s\", terminate-after = 200 }\n"), ); assert_eq!( rules_fired(&root), diff --git a/policy/nextest-slow.rego b/policy/nextest-slow.rego index 7de31ee85..02236afb8 100644 --- a/policy/nextest-slow.rego +++ b/policy/nextest-slow.rego @@ -108,7 +108,7 @@ config := ".config/nextest.toml" # only be on a change that loosens the bound — never on the tree it inherits, # which is the shape `fixture-forks.rego` records as the one that gets an # exception written for it "and the exception is what rots". -ceiling_seconds := 300 +ceiling_seconds := 1200 # The committed runner config, by line index — the index is load-bearing, because # which SECTION a `slow-timeout` sits under is what says whether it is the default @@ -243,11 +243,11 @@ deny contains finding if { tree(lines) := {"tree": {"lines": lines}} -armed := ["[profile.default]", "slow-timeout = { period = \"10s\", terminate-after = 30 }"] +armed := ["[profile.default]", "slow-timeout = { period = \"10s\", terminate-after = 120 }"] report_only := ["[profile.default]", "slow-timeout = \"10s\""] -raised := ["[profile.default]", "slow-timeout = { period = \"30s\", terminate-after = 30 }"] +raised := ["[profile.default]", "slow-timeout = { period = \"30s\", terminate-after = 60 }"] minutes := ["[profile.default]", "slow-timeout = { period = \"2m\", terminate-after = 3 }"] @@ -279,7 +279,7 @@ test_a_period_without_terminate_after_is_refused if { test_a_small_period_with_a_large_multiplier_is_refused if { count(violation) == 1 with input as tree({".config/nextest.toml": [ "[profile.default]", - "slow-timeout = { period = \"10s\", terminate-after = 100 }", + "slow-timeout = { period = \"10s\", terminate-after = 200 }", ]}) }