diff --git a/Cargo.lock b/Cargo.lock index 5477a9d0..cd090559 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5178,7 +5178,7 @@ dependencies = [ [[package]] name = "sentrix" -version = "2.2.39" +version = "2.2.40" dependencies = [ "aes-gcm", "alloy-consensus", @@ -5226,7 +5226,7 @@ dependencies = [ [[package]] name = "sentrix-bft" -version = "2.2.39" +version = "2.2.40" dependencies = [ "bincode", "hex", @@ -5254,7 +5254,7 @@ dependencies = [ [[package]] name = "sentrix-core" -version = "2.2.39" +version = "2.2.40" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "sentrix-evm" -version = "2.2.39" +version = "2.2.40" dependencies = [ "alloy-primitives", "hex", @@ -5301,7 +5301,7 @@ dependencies = [ [[package]] name = "sentrix-faucet" -version = "2.2.39" +version = "2.2.40" dependencies = [ "anyhow", "axum", @@ -5325,14 +5325,14 @@ dependencies = [ [[package]] name = "sentrix-fork-heights" -version = "2.2.39" +version = "2.2.40" dependencies = [ "tracing", ] [[package]] name = "sentrix-grpc" -version = "2.2.39" +version = "2.2.40" dependencies = [ "async-stream", "bincode", @@ -5351,7 +5351,7 @@ dependencies = [ [[package]] name = "sentrix-network" -version = "2.2.39" +version = "2.2.40" dependencies = [ "async-trait", "bincode", @@ -5369,7 +5369,7 @@ dependencies = [ [[package]] name = "sentrix-nft" -version = "2.2.39" +version = "2.2.40" dependencies = [ "hex", "serde", @@ -5380,7 +5380,7 @@ dependencies = [ [[package]] name = "sentrix-node" -version = "2.2.39" +version = "2.2.40" dependencies = [ "anyhow", "axum", @@ -5406,14 +5406,14 @@ dependencies = [ [[package]] name = "sentrix-precompiles" -version = "2.2.39" +version = "2.2.40" dependencies = [ "alloy-primitives", ] [[package]] name = "sentrix-primitives" -version = "2.2.39" +version = "2.2.40" dependencies = [ "hex", "proptest", @@ -5428,7 +5428,7 @@ dependencies = [ [[package]] name = "sentrix-prom-exporter" -version = "2.2.39" +version = "2.2.40" dependencies = [ "http-body-util", "hyper", @@ -5457,7 +5457,7 @@ dependencies = [ [[package]] name = "sentrix-rpc" -version = "2.2.39" +version = "2.2.40" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5490,14 +5490,14 @@ dependencies = [ [[package]] name = "sentrix-rpc-types" -version = "2.2.39" +version = "2.2.40" dependencies = [ "serde_json", ] [[package]] name = "sentrix-staking" -version = "2.2.39" +version = "2.2.40" dependencies = [ "sentrix-primitives", "serde", @@ -5507,7 +5507,7 @@ dependencies = [ [[package]] name = "sentrix-storage" -version = "2.2.39" +version = "2.2.40" dependencies = [ "bincode", "libmdbx", @@ -5522,7 +5522,7 @@ dependencies = [ [[package]] name = "sentrix-trie" -version = "2.2.39" +version = "2.2.40" dependencies = [ "bincode", "blake3", @@ -5539,7 +5539,7 @@ dependencies = [ [[package]] name = "sentrix-wallet" -version = "2.2.39" +version = "2.2.40" dependencies = [ "aes-gcm", "argon2", @@ -5559,7 +5559,7 @@ dependencies = [ [[package]] name = "sentrix-wire" -version = "2.2.39" +version = "2.2.40" dependencies = [ "bincode", "secp256k1 0.31.1", diff --git a/Cargo.toml b/Cargo.toml index 6d0ca668..b2c33c6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [".", "crates/sentrix-primitives", "crates/sentrix-wallet", "crates/se # `version.workspace = true`. Same goes for edition/license/repository so # they can't drift across crates. [workspace.package] -version = "2.2.39" +version = "2.2.40" edition = "2024" license = "BUSL-1.1" repository = "https://github.com/sentrix-labs/sentrix" diff --git a/bin/sentrix/src/commands/chain.rs b/bin/sentrix/src/commands/chain.rs index 565a907e..82534032 100644 --- a/bin/sentrix/src/commands/chain.rs +++ b/bin/sentrix/src/commands/chain.rs @@ -255,3 +255,90 @@ pub fn cmd_chain_verify_deep() -> anyhow::Result<()> { anyhow::bail!("trie ↔ AccountDB inconsistency detected"); } } + +/// Reclaim trie storage by deleting nodes/values unreachable from the last +/// `keep` committed roots — the RACE-FREE counterpart to the background prune. +/// +/// MUST run with the node STOPPED. MDBX is single-writer, and concurrent block +/// commits are exactly what make the background prune delete still-live nodes +/// (the recurring "missing node" stalls — which is why the background prune is +/// now off by default). With the node quiesced, the live-set walk reads a +/// consistent chain.db and only genuine orphans are removed. +/// +/// Operator runbook: halt the validator (verify `pgrep sentrix` is empty), +/// run `sentrix chain prune`, restart. No fork risk — deleting unreachable +/// trie nodes does not change the state_root, which only commits reachable +/// nodes. Safe to run on a single peer (unlike reset-trie). +pub fn cmd_chain_prune(keep: u64) -> anyhow::Result<()> { + use std::sync::Arc; + use std::time::Duration; + + let storage = Storage::open(&get_db_path())?; + if !storage.has_blockchain() { + anyhow::bail!("Chain not initialized."); + } + let height = storage + .load_height() + .map_err(|e| anyhow::anyhow!("reading chain height: {e}"))?; + + // Precondition 1 — the trie must already be PERSISTED. `init_trie` would + // otherwise backfill it from AccountDB; a backfilled trie has a different + // node shape than the incrementally-built one (the 2026-04-21 reset-trie + // fork class), so a maintenance prune must never trigger that rebuild. + // Require the trie root for the current height to be present on disk. + let mdbx = storage.mdbx_arc(); + if height > 0 && !storage.has_persisted_trie_root(height) { + anyhow::bail!( + "Refusing prune: no persisted trie root at height {height}. \ + `chain prune` operates on the EXISTING trie and must not trigger \ + a backfill rebuild (which produces a different node shape — see \ + reset-trie). Boot the node normally once so the trie persists, \ + then halt and re-run." + ); + } + + // Precondition 2 — the node must be STOPPED. MDBX serialises writers, so a + // live validator committing concurrently would reintroduce the exact + // prune-vs-commit race this command exists to avoid. Detect a running node + // via height-stability (the repo's portable check — see + // validator.rs::ensure_chain_not_advancing): a producing node advances the + // persisted height. Sample across > the 5s poll-persist interval. + let h0 = storage.load_height().unwrap_or(height); + std::thread::sleep(Duration::from_secs(7)); + let h1 = storage.load_height().unwrap_or(h0); + if h1 != h0 { + if std::env::var("SENTRIX_ALLOW_ONLINE_PRUNE").map(|v| v == "1").unwrap_or(false) { + tracing::warn!( + "chain prune: height advanced {h0} -> {h1} (node running) — proceeding \ + because SENTRIX_ALLOW_ONLINE_PRUNE=1; this can race live commits" + ); + } else { + anyhow::bail!( + "Refusing prune: chain height advanced {h0} -> {h1} during the check — \ + a validator is producing blocks against this chain.db. Pruning \ + concurrently with commits is exactly the race that deletes live \ + nodes. Stop the node (systemctl stop / docker stop), verify \ + `pgrep sentrix` is empty, then re-run. (Override for a rare \ + recovery: SENTRIX_ALLOW_ONLINE_PRUNE=1.)" + ); + } + } + + let mut bc = storage + .load_blockchain()? + .ok_or_else(|| anyhow::anyhow!("Chain not initialized."))?; + bc.init_trie(Arc::clone(&mdbx))?; + let trie = bc + .state_trie + .as_ref() + .ok_or_else(|| anyhow::anyhow!("trie not initialised"))?; + + println!("Offline trie prune at height {height}, keeping the last {keep} roots."); + let (roots, gc) = trie.prune_offline(keep)?; + if roots == 0 { + println!("Nothing to prune (fewer than {keep} retained roots, or already lean)."); + } else { + println!("Pruned: retired {roots} old roots, GC'd {gc} nodes/values."); + } + Ok(()) +} diff --git a/bin/sentrix/src/main.rs b/bin/sentrix/src/main.rs index e87b7953..43647205 100644 --- a/bin/sentrix/src/main.rs +++ b/bin/sentrix/src/main.rs @@ -589,6 +589,17 @@ enum ChainCommands { /// 2026-04-25 incident root cause). Run with the node STOPPED. Exits 0 if /// consistent, non-zero with a per-address report if any mismatches found. VerifyDeep, + /// Reclaim trie storage: delete trie nodes/values unreachable from the last + /// `--keep` committed roots. The RACE-FREE prune — run with the node STOPPED. + /// The background prune is off by default because it races block apply and + /// can delete still-live nodes ("missing node" stalls); this offline path + /// has no concurrency so it removes only genuine orphans. Safe on a single + /// peer (does not change state_root, unlike reset-trie). + Prune { + /// Number of recent trie root versions to retain (default 1000). + #[arg(long, default_value_t = 1000)] + keep: u64, + }, } #[derive(Subcommand)] @@ -821,6 +832,7 @@ async fn main() -> anyhow::Result<()> { i_understand_divergence_risk, } => commands::chain::cmd_chain_reset_trie(i_understand_divergence_risk)?, ChainCommands::VerifyDeep => commands::chain::cmd_chain_verify_deep()?, + ChainCommands::Prune { keep } => commands::chain::cmd_chain_prune(keep)?, }, Commands::Token { action } => match action { diff --git a/crates/sentrix-core/src/blockchain.rs b/crates/sentrix-core/src/blockchain.rs index 8e8308d6..2c277b55 100644 --- a/crates/sentrix-core/src/blockchain.rs +++ b/crates/sentrix-core/src/blockchain.rs @@ -1540,36 +1540,40 @@ mod tests { assert_eq!(bc.mempool_size(), 1); } - // ── Archive-mode opt-in: SENTRIX_DISABLE_TRIE_PRUNE ───────── + // ── Background prune gate: OFF by default ─────────────────── - /// `crate::blockchain_trie_ops::trie_prune_disabled()` reflects exactly the env var state. - /// Default off; set "1" to enable; other values (empty, "true", - /// "yes", "0") all map to disabled-flag-off-prune-still-runs. + /// `background_prune_enabled()` is OFF by default; it requires + /// `SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE=1` AND no + /// `SENTRIX_DISABLE_TRIE_PRUNE=1` force-disable. #[test] - fn test_trie_prune_disabled_env_var() { + fn test_background_prune_enabled_env_var() { let _guard = crate::test_util::env_test_lock(); + let enabled = || crate::blockchain_trie_ops::background_prune_enabled(); unsafe { - // Baseline: unset → prune runs (predicate false). + std::env::remove_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE"); std::env::remove_var("SENTRIX_DISABLE_TRIE_PRUNE"); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); + // Default: OFF (the racy background prune does not run). + assert!(!enabled()); - // Strict "1" → archive mode on. + // Opt in with strict "1". + std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", "1"); + assert!(enabled()); + + // Non-"1" values do not activate (no silent enable). + for v in ["", "true", "yes", "0"] { + std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", v); + assert!(!enabled(), "value {v:?} must not enable background prune"); + } + + // Legacy force-disable wins even when enable is set. + std::env::set_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE", "1"); std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "1"); - assert!(crate::blockchain_trie_ops::trie_prune_disabled()); - - // Anything else → treated as off (no silent activation). - std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", ""); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); - std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "true"); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); - std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "yes"); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); - std::env::set_var("SENTRIX_DISABLE_TRIE_PRUNE", "0"); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); + assert!(!enabled(), "force-disable must override enable"); // Cleanup so other tests see a clean env. + std::env::remove_var("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE"); std::env::remove_var("SENTRIX_DISABLE_TRIE_PRUNE"); - assert!(!crate::blockchain_trie_ops::trie_prune_disabled()); + assert!(!enabled()); } } diff --git a/crates/sentrix-core/src/blockchain_trie_ops.rs b/crates/sentrix-core/src/blockchain_trie_ops.rs index 6b28d500..8813007a 100644 --- a/crates/sentrix-core/src/blockchain_trie_ops.rs +++ b/crates/sentrix-core/src/blockchain_trie_ops.rs @@ -803,15 +803,19 @@ impl Blockchain { const TRIE_PRUNE_EVERY: u64 = 5000; const TRIE_KEEP_VERSIONS: u64 = 1000; - // Archive-mode opt-in: when SENTRIX_DISABLE_TRIE_PRUNE=1 is set - // in the environment, the periodic prune skips entirely. The - // node accumulates every historical trie version, enabling - // state-at-past-block queries (eth_call at historic h, bridge - // proofs, explorer historical analytics). Off by default — only - // the dedicated archive fullnode sets this; validators stay - // lean. Predicate matches SENTRIX_APPLY_PROFILE's "1"-only - // semantics (block_executor.rs:635) for consistency. - if trie_prune_disabled() { + // Background prune is OFF by default (2026-06-06). It runs on a + // background thread concurrently with block apply; a node committed or + // content-addressed-resurfaced during the multi-minute walk is absent + // from the frozen live-set and gets deleted as an orphan — the + // recurring "missing node" stalls. Five partial fixes (#711/#714/#791/ + // #798) narrowed but never closed the window; a truly race-free + // background prune needs walk+delete in one MDBX RW txn (chain-blocking + // write lock) or refcounting (own project). Until then the fleet + // reclaims trie storage safely via `sentrix chain prune` during a + // maintenance halt (no concurrency → no race). Opt back into the racy + // background path only with SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE=1 + // (and the legacy SENTRIX_DISABLE_TRIE_PRUNE=1 still force-disables). + if !background_prune_enabled() { return; } @@ -882,22 +886,18 @@ impl Blockchain { /// as the existing "failed prune" semantics documented above. static PRUNE_RUNNING: AtomicBool = AtomicBool::new(false); -/// Archive-mode opt-in. When `SENTRIX_DISABLE_TRIE_PRUNE=1` is set in -/// the environment, [`Blockchain::maybe_prune_trie`] returns immediately -/// without scheduling a prune. The node accumulates every historical -/// trie version forever — enabling state-at-past-block queries -/// (`eth_call` at historic h, bridge proofs, explorer historical -/// analytics) at the cost of unbounded disk growth. -/// -/// Default off. Production validators leave this unset and keep the -/// rolling `TRIE_KEEP_VERSIONS = 1000` window. Dedicated archive -/// fullnodes set this flag. -/// -/// Match SENTRIX_APPLY_PROFILE's strict "1" semantics (any other value -/// is treated as off) so accidental `=true` / `=yes` / empty-value -/// settings don't silently activate the archive path. -pub(crate) fn trie_prune_disabled() -> bool { - std::env::var_os("SENTRIX_DISABLE_TRIE_PRUNE").is_some_and(|v| v == "1") +/// Whether the racy BACKGROUND prune is enabled. Default OFF (2026-06-06): +/// the background prune deletes live nodes under concurrency (the recurring +/// "missing node" class) and five partial fixes never closed the window, so +/// the safe path is now `sentrix chain prune` during a maintenance halt +/// (no concurrency → no race). Returns true only when +/// `SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE=1` is set AND the legacy +/// `SENTRIX_DISABLE_TRIE_PRUNE=1` force-disable is NOT set (the latter still +/// wins, for back-compat with archive-node ops scripts). Strict "1" semantics +/// (matching SENTRIX_APPLY_PROFILE) so `=true`/`=yes`/empty don't activate it. +pub(crate) fn background_prune_enabled() -> bool { + std::env::var_os("SENTRIX_ENABLE_BACKGROUND_TRIE_PRUNE").is_some_and(|v| v == "1") + && std::env::var_os("SENTRIX_DISABLE_TRIE_PRUNE").is_none_or(|v| v != "1") } const TESTNET_CHAIN_ID: u64 = 7120; diff --git a/crates/sentrix-core/src/storage.rs b/crates/sentrix-core/src/storage.rs index c97fafbe..0264f2ad 100644 --- a/crates/sentrix-core/src/storage.rs +++ b/crates/sentrix-core/src/storage.rs @@ -32,6 +32,21 @@ impl Storage { self.chain.mdbx_arc() } + /// Whether a trie root is already persisted for `height` (the trie tables + /// hold this height's state). Lets maintenance commands like + /// `sentrix chain prune` refuse to run when they would otherwise trigger + /// an `init_trie` backfill rebuild (different node shape — the reset-trie + /// fork class) instead of operating on the existing trie. + pub fn has_persisted_trie_root(&self, height: u64) -> bool { + self.mdbx_arc() + .get( + sentrix_storage::tables::TABLE_TRIE_ROOTS, + &height.to_be_bytes(), + ) + .map(|o| o.is_some()) + .unwrap_or(false) + } + // ── Blockchain state (everything except blocks) ────── pub fn save_blockchain(&self, blockchain: &Blockchain) -> SentrixResult<()> { diff --git a/crates/sentrix-trie/src/tree.rs b/crates/sentrix-trie/src/tree.rs index 4ce77df8..162a9d37 100644 --- a/crates/sentrix-trie/src/tree.rs +++ b/crates/sentrix-trie/src/tree.rs @@ -699,6 +699,52 @@ impl SentrixTrie { Ok((roots_pruned, nodes_gc + values_gc)) } + /// Offline (quiesced-node) prune — the RACE-FREE GC path. + /// + /// MUST run only when the node is stopped (no concurrent commits). Then the + /// live-set walk reads a consistent MDBX, nothing is re-committed mid-walk, + /// and orphan deletion cannot hit a still-live node — so the immediate + /// (non-generational, non-augmented) delete is safe. + /// + /// This is the safe counterpart to [`Self::prune`], which runs on a + /// background thread concurrently with block apply. That concurrency is the + /// root of the recurring "missing node" class (a node committed/resurfaced + /// during the walk is absent from the frozen live-set and gets deleted); + /// five partial fixes narrowed but never closed the window, so the + /// background prune is now off by default (see `maybe_prune_trie`) and the + /// fleet reclaims trie storage by running `sentrix chain prune` during a + /// maintenance halt instead. Same walk + keep-window semantics as `prune`, + /// minus the racy augment and the generational tombstone deferral. + pub fn prune_offline(&self, keep_versions: u64) -> SentrixResult<(usize, usize)> { + let roots_pruned = self + .cache + .storage + .prune_old_roots(self.version, keep_versions)?; + if roots_pruned == 0 { + return Ok((0, 0)); + } + + let mut live = std::collections::HashSet::new(); + self.collect_reachable(self.root, 0, &mut live)?; + let cutoff = self.version.saturating_sub(keep_versions); + for version in (cutoff + 1)..=self.version { + if let Some(root) = self.cache.storage.load_root(version)? + && !live.contains(&root) + { + self.collect_reachable(root, 0, &mut live)?; + } + } + // No augment_live_to_latest: a quiesced node has no roots past + // self.version. Immediate combined delete is safe with no concurrency. + let gc = self.cache.storage.gc_orphaned_nodes(&live)?; + tracing::info!( + "trie prune (offline): removed {} old roots, GC'd {} nodes/values", + roots_pruned, + gc + ); + Ok((roots_pruned, gc)) + } + /// Walk every committed root in `[from_version, on-disk latest]` not yet in /// `live` and add its reachable hashes. Returns the on-disk latest version /// observed. Keeps the live set current across the long nodes/values GC @@ -1040,6 +1086,43 @@ mod tests { ); } + /// Offline prune: single immediate call reclaims orphans AND keeps every + /// node reachable from the current root. The `get` assertion is the + /// regression guard the production failure needed — a prune that deletes a + /// live node would make this read fail with "missing node". + #[test] + fn test_prune_offline_keeps_reachable_deletes_orphans() { + let (_dir, mdbx) = temp_mdbx(); + let mut trie = SentrixTrie::open(Arc::clone(&mdbx), 0).unwrap(); + let k = address_to_key("0xaaaa"); + + // v1 insert, v2 update same key → v1's old leaf becomes unreachable. + trie.insert(&k, &account_value_bytes(100, 0)).unwrap(); + let _ = trie.commit(1).unwrap(); + trie.insert(&k, &account_value_bytes(200, 1)).unwrap(); + let _ = trie.commit(2).unwrap(); + let nodes_before = mdbx + .count(sentrix_storage::tables::TABLE_TRIE_NODES) + .unwrap(); + + // Offline prune keep=0: immediate, single call (no generational defer). + let (roots, gc) = trie.prune_offline(0).unwrap(); + assert!(roots >= 1, "must retire at least one old root"); + assert!(gc >= 1, "must GC v1's orphaned leaf"); + let nodes_after = mdbx + .count(sentrix_storage::tables::TABLE_TRIE_NODES) + .unwrap(); + assert!(nodes_after < nodes_before, "node count must drop"); + + // The live value MUST survive — proves no live node was deleted. + let v = trie.get(&k).unwrap().unwrap(); + assert_eq!( + account_value_decode(&v).unwrap().0, + 200, + "current value must survive offline prune (no live-node deletion)" + ); + } + /// T-D: open with a custom LRU capacity (small cache, still functionally correct). #[test] fn test_custom_capacity_trie_functional() {