fix(trie): race-free offline prune + background prune off by default

Cargo.toml

@@ -7,7 +7,7 @@ members = [".", "crates/sentrix-primitives", "crates/sentrix-wallet", "crates/se
 # `version.workspace = true`. Same goes for edition/license/repository so
 # they can't drift across crates.
 [workspace.package]
-version = "2.2.39"
+version = "2.2.40"
 edition = "2024"
 license = "BUSL-1.1"
 repository = "https://github.com/sentrix-labs/sentrix"

bin/sentrix/src/commands/chain.rs

@@ -255,3 +255,90 @@ pub fn cmd_chain_verify_deep() -> anyhow::Result<()> {
         anyhow::bail!("trie ↔ AccountDB inconsistency detected");
     }
 }
+
+/// Reclaim trie storage by deleting nodes/values unreachable from the last
+/// `keep` committed roots — the RACE-FREE counterpart to the background prune.
+///
+/// MUST run with the node STOPPED. MDBX is single-writer, and concurrent block
+/// commits are exactly what make the background prune delete still-live nodes
+/// (the recurring "missing node" stalls — which is why the background prune is
+/// now off by default). With the node quiesced, the live-set walk reads a
+/// consistent chain.db and only genuine orphans are removed.
+///
+/// Operator runbook: halt the validator (verify `pgrep sentrix` is empty),
+/// run `sentrix chain prune`, restart. No fork risk — deleting unreachable
+/// trie nodes does not change the state_root, which only commits reachable
+/// nodes. Safe to run on a single peer (unlike reset-trie).
+pub fn cmd_chain_prune(keep: u64) -> anyhow::Result<()> {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    let storage = Storage::open(&get_db_path())?;
+    if !storage.has_blockchain() {
+        anyhow::bail!("Chain not initialized.");
+    }
+    let height = storage
+        .load_height()
+        .map_err(|e| anyhow::anyhow!("reading chain height: {e}"))?;
+
+    // Precondition 1 — the trie must already be PERSISTED. `init_trie` would
+    // otherwise backfill it from AccountDB; a backfilled trie has a different
+    // node shape than the incrementally-built one (the 2026-04-21 reset-trie
+    // fork class), so a maintenance prune must never trigger that rebuild.
+    // Require the trie root for the current height to be present on disk.
+    let mdbx = storage.mdbx_arc();
+    if height > 0 && !storage.has_persisted_trie_root(height) {
+        anyhow::bail!(
+            "Refusing prune: no persisted trie root at height {height}. \
+             `chain prune` operates on the EXISTING trie and must not trigger \
+             a backfill rebuild (which produces a different node shape — see \
+             reset-trie). Boot the node normally once so the trie persists, \
+             then halt and re-run."
+        );
+    }
+
+    // Precondition 2 — the node must be STOPPED. MDBX serialises writers, so a
+    // live validator committing concurrently would reintroduce the exact
+    // prune-vs-commit race this command exists to avoid. Detect a running node
+    // via height-stability (the repo's portable check — see
+    // validator.rs::ensure_chain_not_advancing): a producing node advances the
+    // persisted height. Sample across > the 5s poll-persist interval.
+    let h0 = storage.load_height().unwrap_or(height);
+    std::thread::sleep(Duration::from_secs(7));
+    let h1 = storage.load_height().unwrap_or(h0);
+    if h1 != h0 {
+        if std::env::var("SENTRIX_ALLOW_ONLINE_PRUNE").map(|v| v == "1").unwrap_or(false) {
+            tracing::warn!(
+                "chain prune: height advanced {h0} -> {h1} (node running) — proceeding \
+                 because SENTRIX_ALLOW_ONLINE_PRUNE=1; this can race live commits"
+            );
+        } else {
+            anyhow::bail!(
+                "Refusing prune: chain height advanced {h0} -> {h1} during the check — \
+                 a validator is producing blocks against this chain.db. Pruning \
+                 concurrently with commits is exactly the race that deletes live \
+                 nodes. Stop the node (systemctl stop / docker stop), verify \
+                 `pgrep sentrix` is empty, then re-run. (Override for a rare \
+                 recovery: SENTRIX_ALLOW_ONLINE_PRUNE=1.)"
+            );
+        }
+    }
+
+    let mut bc = storage
+        .load_blockchain()?
+        .ok_or_else(|| anyhow::anyhow!("Chain not initialized."))?;
+    bc.init_trie(Arc::clone(&mdbx))?;
+    let trie = bc
+        .state_trie
+        .as_ref()
+        .ok_or_else(|| anyhow::anyhow!("trie not initialised"))?;
+
+    println!("Offline trie prune at height {height}, keeping the last {keep} roots.");
+    let (roots, gc) = trie.prune_offline(keep)?;
+    if roots == 0 {
+        println!("Nothing to prune (fewer than {keep} retained roots, or already lean).");
+    } else {
+        println!("Pruned: retired {roots} old roots, GC'd {gc} nodes/values.");
+    }
+    Ok(())
+}

bin/sentrix/src/main.rs

@@ -589,6 +589,17 @@ enum ChainCommands {
     /// 2026-04-25 incident root cause). Run with the node STOPPED. Exits 0 if
     /// consistent, non-zero with a per-address report if any mismatches found.
     VerifyDeep,
+    /// Reclaim trie storage: delete trie nodes/values unreachable from the last
+    /// `--keep` committed roots. The RACE-FREE prune — run with the node STOPPED.
+    /// The background prune is off by default because it races block apply and
+    /// can delete still-live nodes ("missing node" stalls); this offline path
+    /// has no concurrency so it removes only genuine orphans. Safe on a single
+    /// peer (does not change state_root, unlike reset-trie).
+    Prune {
+        /// Number of recent trie root versions to retain (default 1000).
+        #[arg(long, default_value_t = 1000)]
+        keep: u64,
+    },
 }
 
 #[derive(Subcommand)]
@@ -821,6 +832,7 @@ async fn main() -> anyhow::Result<()> {
                 i_understand_divergence_risk,
             } => commands::chain::cmd_chain_reset_trie(i_understand_divergence_risk)?,
             ChainCommands::VerifyDeep => commands::chain::cmd_chain_verify_deep()?,
+            ChainCommands::Prune { keep } => commands::chain::cmd_chain_prune(keep)?,
         },
 
         Commands::Token { action } => match action {

crates/sentrix-core/src/blockchain.rs

@@ -1540,36 +1540,40 @@ mod tests {
         assert_eq!(bc.mempool_size(), 1);
     }
 
-    // ── Archive-mode opt-in: SENTRIX_DISABLE_TRIE_PRUNE ─────────
+    // ── Background prune gate: OFF by default ───────────────────
 
-    /// `crate::blockchain_trie_ops::trie_prune_disabled()` reflects exactly the env var state.
-    /// Default off; set "1" to enable; other values (empty, "true",
-    /// "yes", "0") all map to disabled-flag-off-prune-still-runs.
+    /// `background_prune_enabled()` is OFF by default; it requires
+    /// `SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE=1` AND no
+    /// `SENTRIX_DISABLE_TRIE_PRUNE=1` force-disable.
     #[test]
-    fn test_trie_prune_disabled_env_var() {
+    fn test_background_prune_enabled_env_var() {
         let _guard = crate::test_util::env_test_lock();
+        let enabled = || crate::blockchain_trie_ops::background_prune_enabled();
         unsafe {
-            // Baseline: unset → prune runs (predicate false).
+            std::env::remove_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE");
             std::env::remove_var("SENTRIX_DISABLE_TRIE_PRUNE");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
+            // Default: OFF (the racy background prune does not run).
+            assert!(!enabled());
 
-            // Strict "1" → archive mode on.
+            // Opt in with strict "1".
+            std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", "1");
+            assert!(enabled());
+
+            // Non-"1" values do not activate (no silent enable).
+            for v in ["", "true", "yes", "0"] {
+                std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", v);
+                assert!(!enabled(), "value {v:?} must not enable background prune");
+            }
+
+            // Legacy force-disable wins even when enable is set.
+            std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", "1");
             std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "1");
-            assert!(crate::blockchain_trie_ops::trie_prune_disabled());
-
-            // Anything else → treated as off (no silent activation).
-            std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
-            std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "true");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
-            std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "yes");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
-            std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "0");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
+            assert!(!enabled(), "force-disable must override enable");
 
             // Cleanup so other tests see a clean env.
+            std::env::remove_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE");
             std::env::remove_var("SENTRIX_DISABLE_TRIE_PRUNE");
-            assert!(!crate::blockchain_trie_ops::trie_prune_disabled());
+            assert!(!enabled());
         }
     }