diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddace36..cea3d36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,19 @@ jobs: # requested in mise.toml don't survive every cache round-trip). # Idempotent and ~2s when the components are already present. - run: rustup component add clippy rustfmt + # Unformatted code (Rust or YAML) fails the build rather than drifting + # until the next `mise run format` reformats someone else's diff. + - run: mise run format:check - run: cargo clippy --all-targets --all-features -- -D warnings + # Broken intra-doc links fail too (docs.rs renders them). + - run: cargo doc --no-deps --all-features + env: + RUSTDOCFLAGS: -D warnings + # Each optional feature compiles alone with its tests: shared test + # infrastructure must not quietly depend on another feature. + - run: cargo check --tests --features fjall + - run: cargo check --tests --features rocksdb + - run: cargo check --tests --features transport # The full feature matrix, or the fjall/rocksdb conformance suites and # the transport tier silently skip (default features exclude them all). # Integration tests boot real nats-server + minio binaries (installed diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a664f55..3153312 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -182,14 +182,20 @@ Every non-`_from` watch is a **state-sync** stream (NATS `DeliverPolicy::LastPer The cursor is the NATS stream sequence number at the last checkpoint. On restart, pass it to `watch_all_from()` to subscribe at `cursor+1` — only the delta arrives, not the full history. -When the cursor expires (NATS retention window evicted those records), `CursorExpired` is returned. The fallback `watch_all()` re-list re-delivers the current value of every live key, but it cannot cover keys **deleted during the gap whose delete markers were also evicted** — those need synthetic `Delete` events diffed from prior state. On a bucket whose retention never evicts current values, `watch_applied` does this automatically (see below), and a raw-API caller hand-rolls the same diff with `Snapshot::stale_keys()`: +When the cursor expires (NATS retention window evicted those records), `CursorExpired` is returned. The fallback `watch_all()` re-list re-delivers the current value of every live key, but it cannot cover keys **deleted during the gap whose delete markers were also evicted** — those need synthetic `Delete` events diffed from prior state. On a bucket whose retention never evicts current values, `watch_applied` does this automatically (see below), and a raw-API caller hand-rolls the same diff with `Snapshot::stale_keys()`, which returns `None` when the bucket's retention says the listing isn't the truth (the same planner as `ExpiryRepair::Relist`): ```rust match watcher.watch_all_from(&snap.cursor, tx).await { Ok(()) => {} Err(KvError::CursorExpired) => { let live = reader.keys("").await?; - for key in snap.stale_keys(live.iter().map(|s| s.as_str())) { + let Some(stale) = snap.stale_keys(live.iter().map(|s| s.as_str()), watcher.retention().await?) + else { + // Retention evicts current values: a missing key may have aged + // out. Restore from an artifact instead (ExpiryRepair::Restore). + return Err(KvError::CursorExpired.into()); + }; + for key in stale { cache.remove(key); } watcher.watch_all(tx).await?; @@ -224,7 +230,7 @@ The one exception: an update carrying the **unknown** version (an unparseable AC **Flush triggers.** A batch flushes when any of these fires: the `window` elapses, `batch.len()` reaches `config.max`, a shutdown is signalled, or the channel closes with a pending batch (the remainder is flushed before returning). -**Transient store failures re-queue, not drop.** If `store.apply()` returns an error, the raw batch is prepended to the next flush's accumulation and the watch continues. The streak counter increments; at 16 consecutive failures the watch fail-stops with `KvError::WatchError`. Dropping the failed batch and continuing was the shipped behavior until `transient_store_failure_never_leaves_a_cursor_gap` reproduced the bug: a transient failure followed by a successful flush advanced the cursor over a hole that survived every restart, because the restart re-folds from the advanced cursor, skipping the missing range. Cursor authority requires the store's cursor and contents to advance together, always (`applied.rs:303–394`, `tests/model_applied.rs`). +**Transient store failures re-queue, not drop.** If `store.apply()` returns an error, the raw batch is prepended to the next flush's accumulation and the watch continues. The streak counter increments; at 16 consecutive failures the watch fail-stops with `KvError::WatchError`. Dropping the failed batch and continuing was the shipped behavior until `transient_store_failure_never_leaves_a_cursor_gap` reproduced the bug: a transient failure followed by a successful flush advanced the cursor over a hole that survived every restart, because the restart re-folds from the advanced cursor, skipping the missing range. Cursor authority requires the store's cursor and contents to advance together, always (`Fold::flush` in `applied.rs`, `tests/model_applied.rs`). **Cursor-expiry repair.** The cursor expires at resume (`CursorExpired` from the `*_from` watch) or mid-watch (the live floor guard ends an All-scope watch with `CursorExpired` when retention overruns it). Both take one path. Correct means **every write minus every real delete** — not "whatever NATS still lists"; the two agree only when retention never evicts current values. The repair (`ExpiryRepair`, `src/repair.rs`) is chosen accordingly: @@ -292,7 +298,18 @@ Every NATS operation is wrapped in `timed()` (30 s). Without it, a CLOSE_WAIT co ### Machine-Checked Protocol Kernels (`protocol.rs`) -Four pure-function guards live in `protocol.rs`. Production code and the Stateright exhaustive model checker (`tests/model.rs`) call the **same functions** — not two hand-synchronized copies. A change to any guard is re-verified against the full bounded state space on the next `cargo test --test model`. Mutation tests prove each guard is load-bearing: substituting a broken variant produces a counterexample. +Every decision the protocol's correctness rests on is a pure function in `protocol.rs`. Production code and the Stateright models (`tests/model*.rs`) call the **same functions** — not hand-synchronized copies. A change to any of them is re-verified against the full bounded state space on the next model run. A model expresses a mutation by forging a kernel's *input* (e.g. the listing is "the truth" on an evicting bucket), never by re-implementing the decision; each mutation must produce a counterexample. + +| Kernel | Production | Models | +| --- | --- | --- | +| `pointer_publish_allowed` | `transport` pointer swap | `model.rs`, `model_fleet.rs` | +| `payload_prunable` | `transport` prune | `model.rs` | +| `resume_window_ok` | `nats` resume paths (`create_resume`), floor guard | every model | +| `restore_allowed` / `restore_ahead` | `repair::check_restore`, the main loop's ahead re-check | `model.rs`, `model_live_watch.rs`, `model_repair.rs`, `model_fleet.rs` | +| `listing_is_truth` | `repair` (plan, cursor-less start), `Snapshot::stale_keys` | every model's view of retention | +| `plan_repair` | `repair::plan`, `Snapshot::stale_keys` | every model's expiry dispatch | +| `cursorless_start_needs_repair` | `repair::run_watch` | `model_repair.rs`, `model_fleet.rs` | +| `restore_key` | `repair::restore_diff` | `model_repair.rs` | **`pointer_publish_allowed(current: &PointerState, candidate_rank: u64) → bool`** @@ -341,6 +358,18 @@ artifact_revision > local_revision && resume_window_ok(artifact_revision, first_ Machine-checked as: _"bootstrap never silently diverges from every write minus every real delete"_ on an evicting bucket (`tests/model.rs`), and _"a repair never moves the fold backward"_ (`tests/model_live_watch.rs`, where removing the guard is a mutation the checker catches). +**`listing_is_truth(evicts_current_values, first_revision) → bool`** and **`plan_repair(mode, listing_truth) → RepairPlan`** + +Whether "not listed" means "deleted": yes when retention never evicts current values, or when it can but has evicted nothing yet (first retained revision ≤ 1). The planner turns that (or `None`, a backend that can't say) and the armed `ExpiryRepair` into the repair: `Relist` where the listing isn't the truth is `RefuseRelist`; `Auto` re-lists only on `Some(true)` and restores otherwise. + +**`cursorless_start_needs_repair(mode, fold_has_data, listing_truth) → bool`** + +A start with no cursor repairs first when the fold holds data anyway (a re-list never removes what it doesn't deliver), or, with a restore armed, when the listing isn't the truth (a re-list would be incomplete). + +**`restore_key(local, artifact, identical, listed_live) → KeyRestore`** + +The per-key restore decision: take the artifact's entry unless the local one is newer (never a move backward), and delete a key the artifact lacks unless the bucket lists it live (never a phantom delete). `tests/model_repair.rs` catches both pre-fix variants as forged inputs. + ### Export Round State Machine ``` @@ -636,7 +665,7 @@ Machine-checked as three theorems in `tests/model.rs`: published cursor never re ### Why extract protocol guards into `protocol.rs` and share them with the model? -Production code and an exhaustive model checker running the same logic closes the drift gap between proof and implementation. Previously the model had its own inline guard copies; a change to the production guard would not update the model, leaving the proof covering the old variant. Extracting the three guards into `protocol.rs` and importing them from both call sites means every `cargo test --test model` verifies the current production code, not a snapshot. The cost is a hard dependency: `tests/model.rs` imports `pub mod protocol` directly, so the module cannot go private. +Production code and an exhaustive model checker running the same logic closes the drift gap between proof and implementation. Previously the models had their own inline copies; a change to the production decision would not update them, leaving the proof covering the old variant. Extracting every decision into `protocol.rs` and importing it from both call sites means every model run verifies the current production code, not a snapshot. The cost is a hard dependency: the models import `pub mod protocol` directly, so the module cannot go private. ### Why write in sorted key order during compaction? @@ -729,8 +758,8 @@ Production code and an exhaustive model checker running the same logic closes th | -------------------------- | ------------------------------------------------------------------------------------ | | `src/kv.rs` | Core traits (`KvReader`, `KvWriter`, `KvWatcher`, `KvTtl`) and types (`KvEntry`, `KvUpdate`, `VersionToken`, `WatchCursor`, `KvError`) | | `src/stores.rs` | `Connection`, `KvStore`, `StoreConfig`, `StorageType`, `ConnectionCapabilities` | -| `src/nats.rs` | NATS JetStream implementation; bucket creation, scan consumer lifecycle, timeout wrapping, Synadia Cloud workarounds, `check_resume_window` | -| `src/protocol.rs` | Pure-function protocol guards: `pointer_publish_allowed`, `payload_prunable`, `resume_window_ok` — called by both production code and the Stateright model | +| `src/nats.rs` | NATS JetStream implementation; bucket creation, scan consumer lifecycle, timeout wrapping, Synadia Cloud workarounds, resume-window checks (`create_resume`), the live floor guard | +| `src/protocol.rs` | The protocol's decisions as pure functions (see "Machine-Checked Protocol Kernels"), called by both production code and the Stateright models | | `src/snapshot.rs` | `SnapshotStore` trait; append-only log + `AppendLogSnapshot` (default backend): `SnapshotWriter`, `load()`, `replay_log()`, `compact_to_file()` | | `src/snapshot_fjall.rs` | `FjallSnapshot`: on-disk `SnapshotStore` backed by fjall (`feature = "fjall"`) | | `src/snapshot_rocksdb.rs` | `RocksDbSnapshot`: on-disk `SnapshotStore` backed by RocksDB (`feature = "rocksdb"`) | @@ -738,8 +767,8 @@ Production code and an exhaustive model checker running the same logic closes th | `src/artifact.rs` | `ExportManifest`, `ArtifactFile`, BLAKE3 integrity; stage-then-rename discipline; backend `export_to` + `import` (append-log, fjall, RocksDB) | | `src/export_lease.rs` | `ExportLease`, `LeaseGuard`, `LeaseRecord`: fleet-wide at-most-one via embedded-expiry CAS | | `src/transport.rs` | `ObjectStoreTransport`, `ArtifactTransport` trait, `run_export_round`: monotonic pointer swap, multipart upload, prune, content-addressed keys (`feature = "transport"`) | -| `src/applied.rs` | `watch_applied` cursor-after-apply combinator, generic over `SnapshotStore`: `WatchScope`, `BatchConfig`, cursor-expiry repair (drain, key-listing diff, artifact restore), `ExportRequest` handling (stamps the scope) | -| `src/repair.rs` | `ExpiryRepair`, `RestoreSource`, `RestoredFold`; scope coverage, restore checks, the in-scope restore diff | +| `src/applied.rs` | `watch_applied` cursor-after-apply combinator, generic over `SnapshotStore`: `WatchScope`, `BatchConfig`, the select loop, and `Fold` — the batch, applied cursor, and store every change goes through (ingest, repair corrections, flush, `ExportRequest` handling, which stamps the scope) | +| `src/repair.rs` | Cursor-expiry repair, both halves: `ExpiryRepair`, `RestoreSource`, `RestoredFold`; the watch task (`run_watch`: plan, key listing, artifact checks and fetch) and the main loop's side (`fold_in`: drain, settle, key-listing diff, artifact restore into the `Fold`); scope coverage, restore checks, the in-scope restore diff | | `src/lib.rs` | Re-exports all public types; no logic | | `benches/` | Criterion benchmarks: snapshot write/checkpoint/load throughput, batch throughput, ACK subject parsing | | `tests/integration.rs` | NATS JetStream backend integration suite: each test boots its own `nats-server` on a free port; covers bucket create, CAS, watch semantics, cursor resume, and delete reconciliation | @@ -757,7 +786,7 @@ Production code and an exhaustive model checker running the same logic closes th | `tests/model_applied.rs` | Stateright cursor-authority model: delivery → flush → transient failure → crash/restart; `DropFailedBatch` and `ResumeFromMemApplied` mutations pin both pre-fix bug classes | | `tests/model_resync_order.rs` | Stateright resync ack-barrier model: proves synthetic deletes strictly precede re-list puts; `NoAckBarrier` mutation reaches lost-recreate divergence | | `tests/model_live_watch.rs` | Stateright floor-guard model: guarded variant proves the fold always converges to the truth, repaired by the key-listing diff or (evicting bucket) the artifact restore; unguarded variant pins permanent silent divergence; mutations: relist on an evicting bucket, restore without `restore_allowed` | -| `tests/common/mod.rs` | Shared test helpers: ephemeral NATS server, MinIO harness, `ManifestPutCrash` transport injection | +| `tests/common/` | Shared test helpers: `nats.rs` (ephemeral NATS server, also the crate's own unit tests' via `#[path]`), `minio.rs` (MinIO harness), `crash.rs` (`ManifestPutCrash` transport injection, `feature = "transport"`) | ## Configuration diff --git a/src/applied.rs b/src/applied.rs index b60f343..7db2fe5 100644 --- a/src/applied.rs +++ b/src/applied.rs @@ -60,15 +60,11 @@ use std::time::Duration; use tokio::sync::mpsc; use tokio::sync::{oneshot, watch}; -use tracing::{error, info, warn}; +use tracing::warn; use crate::artifact::ExportManifest; -use crate::kv::{KvError, KvReader, KvUpdate, KvWatcher, Retention, WatchCursor}; -use crate::protocol::resume_window_ok; -use crate::repair::{ - ExpiryRepair, RestoreSource, RestoredFold, check_restore, cursor_rank, materialize, - restore_diff, -}; +use crate::kv::{KvError, KvUpdate, KvWatcher, WatchCursor}; +use crate::repair::{self, ExpiryRepair}; use crate::snapshot::{SnapshotError, SnapshotStore}; /// A request, sent into a running [`watch_applied`] loop, to export the fold it @@ -112,7 +108,7 @@ pub enum WatchScope { impl WatchScope { /// The scope as a list of key prefixes (`All` = the empty prefix), for /// callers that enumerate scope contents (live listings, fold ranges). - fn prefixes(&self) -> Vec { + pub(crate) fn prefixes(&self) -> Vec { match self { WatchScope::All => vec![String::new()], WatchScope::Prefix(p) => vec![p.clone()], @@ -121,51 +117,6 @@ impl WatchScope { } } -/// Internal: a cursor-expired repair handoff from the watch task to the main -/// loop, which owns the fold. The watch task stays parked on the ack/reply -/// until the main loop has folded the repair, so the repair is strictly -/// ordered between everything delivered before the expiry and everything the -/// next watch delivers. -enum RepairRequest { - /// The key-listing diff: the bucket's live keys for the watch scope. The - /// main loop applies synthetic deletes for in-scope keys missing from it, - /// then acks so the watch task can start the fallback re-list. - Relist { - live_keys: Vec, - ack: oneshot::Sender<()>, - }, - /// The artifact restore: a verified artifact fold, and the bucket's live - /// keys for the scope (listed after the fetch). The main loop folds the - /// in-scope difference, advances to the artifact's cursor, and replies - /// with it so the watch task resumes from there. - Restore { - restored: RestoredFold, - live: std::collections::HashSet, - reply: oneshot::Sender>, - }, -} - -/// Internal: what the watch task needs to repair an expiry — the caller's -/// chosen repair, and the channel into the main loop that owns the fold. -struct RepairHandle { - mode: ExpiryRepair, - tx: mpsc::Sender>, -} - -/// Internal: the repair the watch task settled on for one expiry. -enum Plan<'a, S> { - /// Fall back to the re-list alone (nothing armed). - ReListOnly, - /// The key-listing diff, then the re-list. - Relist(&'a Arc, &'a mpsc::Sender>), - /// The artifact restore, then a resume from the artifact's cursor. - Restore( - &'a Arc, - &'a Arc>, - &'a mpsc::Sender>, - ), -} - /// Batching policy for [`watch_applied`]. /// /// A flush fires when **either** bound is hit, whichever comes first: `window` @@ -269,12 +220,6 @@ impl Default for BatchConfig { // type and is monomorphized at the call site. Folding them into a builder struct // would either box the closures or force a single generic bundle, losing that. #[allow(clippy::too_many_arguments)] -// The flush macro resets `batch_high`/`batch_deadline` for the next loop -// iteration. At the two flush sites that return immediately afterward (shutdown, -// channel-close) those resets are dead stores — correct, but flagged. The allow -// must sit on the function: a statement-scoped `#[allow]` inside the macro body -// trips the experimental attributes-on-expressions gate (E0658) on stable. -#[allow(unused_assignments)] pub async fn watch_applied( watcher: Arc, scope: WatchScope, @@ -284,7 +229,7 @@ pub async fn watch_applied( // bucket that has already evicted current values) — the hot path never // touches it. repair: R, - mut store: Option, + store: Option, // `Some(rx)` arms an export-request arm in the select loop: each // [`ExportRequest`] is handled between flushes (pending batch flushed // first), so the exported artifact's cursor is the applied cursor (or, @@ -295,9 +240,9 @@ pub async fn watch_applied( // unchanged. mut exports: Option>, config: BatchConfig, - mut parse: P, - mut apply: A, - mut on_applied: O, + parse: P, + apply: A, + on_applied: O, mut shutdown: watch::Receiver, ) -> Result where @@ -311,282 +256,55 @@ where O: FnMut(WatchCursor) + Send, R: Into>, { - // The cursor we'll return. Initialized from the resume position so that a - // watch which receives nothing new still reports the position it resumed - // from as "applied" (it is — everything up to it was applied before the last - // run persisted it). - let mut applied = match &resume { - Some(c) => c.clone(), - None => WatchCursor::none(), - }; - // The scope's prefixes, for the repair diffs against the fold and the // scope recorded in exported artifacts. Cloned out before `scope` moves // into the watch task. - let scope_prefixes = scope.prefixes(); - - // Repair channel, armed only when there is a repair to run AND a store to - // run it against (both repairs edit the fold). The watch task sends the - // repair here and waits for the ack/reply before starting its next watch. - let mode: ExpiryRepair = repair.into(); - // A restore replaces the fold, so asking for one without a fold is a - // contradiction: on a bounded log such a consumer would silently hold - // NATS's retained view (missing everything evicted) while believing it - // repairs. Refuse it up front; `ExpiryRepair::None` is the explicit way - // to accept the retained view. - if store.is_none() - && matches!( - mode, - ExpiryRepair::Restore { .. } | ExpiryRepair::Auto { .. } - ) - { - return Err(KvError::WatchError( - "ExpiryRepair::Restore / ::Auto repair the fold, so they need a store: a consumer \ - without one cannot repair a bounded log. Pass a SnapshotStore, or \ - ExpiryRepair::None to accept NATS's retained view" - .into(), - )); - } - let (repair_handle, mut repairs): ( - Option>, - Option>>, - ) = match mode { - ExpiryRepair::None => (None, None), - mode if store.is_some() => { - let (tx, rx) = mpsc::channel(1); - (Some(RepairHandle { mode, tx }), Some(rx)) - } - _ => (None, None), - }; + let prefixes = scope.prefixes(); + + // The repair channel: armed only when there is a repair to run AND a + // store to run it against (both repairs edit the fold). The watch task + // sends the repair here and waits for the ack/reply before starting its + // next watch. + let (repair, mut repairs) = repair::arm(repair.into(), store.is_some())?; // A fold with data but no cursor can't be re-listed blind (see - // `run_watch`). Only the first entry is read. + // `repair::run_watch`). let unanchored = match &store { - Some(st) if resume.as_ref().is_none_or(WatchCursor::is_none) => fold_has_entries(st) + Some(st) if resume.as_ref().is_none_or(WatchCursor::is_none) => st + .has_entries() .map_err(|e| KvError::WatchError(format!("reading the fold at start failed: {e}")))?, _ => false, }; + // The cursor we'll return. Initialized from the resume position so that a + // watch which receives nothing new still reports the position it resumed + // from as "applied" (it is — everything up to it was applied before the last + // run persisted it). + let applied = resume.clone().unwrap_or_else(WatchCursor::none); + let mut fold = Fold::new(store, parse, apply, on_applied, config.max, applied); + // Spawn the watch task. It owns the cursor-expiry handling so the main loop // only ever sees a clean ordered stream of updates on `rx`, plus repairs. let (tx, mut rx) = mpsc::channel::(config.channel_capacity.max(1)); - let handle = { - let watcher = Arc::clone(&watcher); - tokio::spawn(async move { - run_watch( - watcher.as_ref(), - &scope, - resume, - unanchored, - repair_handle, - tx, - ) - .await - }) + let handle = tokio::spawn(repair::run_watch( + watcher, scope, resume, unanchored, repair, tx, + )); + // A fatal error ends the watch, and the watch task with it. + let stop = |e: KvError| { + handle.abort(); + e }; - // Batch state. - // - // `batch_high` tracks the version of the most recently *received* update - // since the last flush — including updates `parse` rejected. NATS delivers - // in revision order, so the last received is the highest, and advancing the - // cursor to it after a single atomic `apply` is correct: having seen the max - // means we've seen everything below it, and a rejected entry is still - // "nothing to apply", hence covered. Reset to `none()` after every flush. - // Pre-size to the flush bound so no batch ever re-climbs the reallocation - // ladder; `max(1)` only guards a nonsensical `max = 0` config. - let batch_cap = config.max.max(1); - let mut batch: Vec = Vec::with_capacity(batch_cap); - // Raw received updates for the durable `store`, in revision order. Only - // populated when a `store` is present; the store folds the *raw* updates - // (including ones `parse` rejected — they are still part of the bucket's - // state), whereas the parsed `batch` above is the consumer's domain view. - let mut raw_batch: Vec = Vec::new(); - let mut batch_high = WatchCursor::none(); - // Consecutive store-apply failures. A transient failure re-queues its raw - // batch (cursor authority: the store's cursor and contents advance - // together, always); a persistent streak fail-stops before the re-queued - // backlog grows without bound. - const MAX_STORE_APPLY_FAILURES: u32 = 16; - let mut store_fail_streak: u32 = 0; - // `Some` once a batch has opened and the window timer is armed; `None` - // between flushes. Only the armed/idle distinction is read in the loop — - // the absolute instant lives in the pinned `sleep` future below. - let mut batch_deadline: Option = None; - // A single timer future, reset in place each time a batch opens. The old // `tokio::time::sleep(timeout)` lived inside the select arm, so it was // re-created on every loop iteration — one Arc-backed timer-wheel entry // allocated, registered, and immediately dropped per received update. // Pinning one future and `reset`-ing it reuses that single allocation; the - // `if batch_deadline.is_some()` guard keeps it from firing while idle, so - // its initial already-elapsed deadline is never observed. + // `if fold.is_open()` guard keeps it from firing while idle, so its + // initial already-elapsed deadline is never observed. let sleep = tokio::time::sleep(Duration::ZERO); tokio::pin!(sleep); - // Flush the current batch, in order: run the domain `apply` (if non-empty) to - // completion, advance the cursor, fold the raw batch + cursor durably into - // `store`, then fire `on_applied`. The store fold runs on a blocking task - // (its `apply` may block on I/O), moving the store in and taking it back — the - // same offload the append log's compaction always used. A TRANSIENT store - // error re-queues the raw batch for cumulative commit on the next flush - // (the watch continues; the store's cursor never advances past data it - // dropped) and a persistent failure streak is fatal; a panicked - // blocking task drops the store irrecoverably, which breaks the - // resume-after-restart guarantee, so it is surfaced as fatal. - macro_rules! flush { - () => {{ - // Nothing received since the last flush → nothing to do at all. - // (`raw_batch` can be non-empty with no cursor advance only via a - // repair's synthetic updates, which carry no stream position.) - if !batch.is_empty() || !raw_batch.is_empty() || !batch_high.is_none() { - if !batch.is_empty() { - // INVARIANT: apply() runs and RETURNS before any cursor - // advance below. Move the batch out so a panicking apply - // can't leave half-consumed state behind. - // - // `replace` (not `take`) leaves a pre-sized Vec behind so each - // batch after the first doesn't re-climb the reallocation - // ladder (4→8→…→cap). - apply(std::mem::replace(&mut batch, Vec::with_capacity(batch_cap))); - } - let advanced = !batch_high.is_none(); - if advanced { - applied = batch_high.clone(); - } - // A cursor-only advance (nothing raw to fold) still commits - // to the store: a restore's final step moves the cursor to the - // artifact's with no updates left. In the steady state every - // advance carries raw updates, so this changes nothing there. - if (!raw_batch.is_empty() || advanced) - && let Some(mut st) = store.take() - { - let raw = std::mem::take(&mut raw_batch); - // Fold at the post-advance cursor. A repair-only batch - // (synthetic updates) leaves the cursor where it was (they - // are a state correction, not log entries), which is safe: - // an unchanged — possibly expired — cursor only ever re-runs - // the same repair on the next restart. - let cur = applied.clone(); - // Hand the store AND the raw batch back on a clean return: - // a *failed* apply (Ok(Err)) RE-QUEUES the batch so the - // next flush commits it cumulatively — the store's cursor - // and contents always advance together. Dropping the - // failed batch instead lets the NEXT successful flush - // advance the cursor over a hole that survives every - // restart (reproduced by - // `transient_store_failure_never_leaves_a_cursor_gap`). - // Only a *panicked* task (Err) loses the store: fatal. - match tokio::task::spawn_blocking(move || { - let res = st.apply(&raw, &cur); - (st, raw, res) - }) - .await - { - Ok((st, _raw, Ok(()))) => { - store = Some(st); - store_fail_streak = 0; - } - Ok((st, raw, Err(e))) => { - store_fail_streak += 1; - if store_fail_streak >= MAX_STORE_APPLY_FAILURES { - // A persistently failing store would otherwise - // grow the re-queued batch without bound while - // the fold silently stales. Fail-stop: the - // restart refolds the tail from the store's - // last good cursor. - warn!(error = %e, streak = store_fail_streak, - "snapshot store apply failing persistently; aborting watch"); - handle.abort(); - return Err(KvError::WatchError(format!( - "snapshot store apply failed {store_fail_streak} consecutive times: {e}" - ))); - } - warn!(error = %e, streak = store_fail_streak, - "snapshot store apply failed; batch re-queued for the next flush"); - store = Some(st); - // Prepend: the failed range precedes anything - // received since (stream order is preserved for - // the eventual cumulative commit). - let newer = std::mem::replace(&mut raw_batch, raw); - raw_batch.extend(newer); - } - Err(e) => { - warn!(error = %e, "snapshot store task panicked; aborting watch"); - handle.abort(); - return Err(KvError::WatchError(format!( - "snapshot store task panicked: {e}" - ))); - } - } - } - if advanced { - on_applied(applied.clone()); - batch_high = WatchCursor::none(); - } - } - batch_deadline = None; - }}; - } - - // Take one update off the watch channel into the pending batch. - macro_rules! ingest { - ($u:expr) => {{ - let u: KvUpdate = $u; - // Cursor authority: every received update bumps the pending - // high-water, regardless of whether `parse` keeps it — but only - // when it carries a real position. An unknown version (e.g. an - // unparseable ACK subject on the hand-built multi-prefix consumer - // path) must neither mint a fake cursor nor clobber the real high - // from earlier in the batch; skipping it under-advances at worst, - // and re-delivery on resume is idempotent. - if !u.version().is_unknown() { - batch_high = WatchCursor::from_version(u.version().clone()); - } - - // Buffer the raw update for the durable store fold (which commits - // the whole batch + cursor atomically on flush). Done before - // `parse` consumes `u` by reference, and only when a store is - // present so the no-persistence path keeps its zero-copy cost. - if store.is_some() { - raw_batch.push(u.clone()); - } - - if let Some(parsed) = parse(&u) { - batch.push(parsed); - } - - // Arm the window on the first received update of a batch — even a - // parse-rejected one, so the cursor advances within `window` even - // through a run of irrelevant keys. Reset the pinned timer to the - // new deadline rather than allocating a fresh `Sleep`. - if batch_deadline.is_none() { - let deadline = tokio::time::Instant::now() + config.window; - sleep.as_mut().reset(deadline); - batch_deadline = Some(deadline); - } - - // Flush on a full parsed batch, or — when persisting — a full raw - // batch, so a window packed with parse-rejected updates can't grow - // `raw_batch` without bound before the window elapses. - if batch.len() >= config.max || raw_batch.len() >= config.max { - flush!(); - } - }}; - } - - // Fail the watch from inside a repair: the fold is either unchanged or - // part-way through a repair that commits under its old (expired) cursor, - // so the caller's restart re-runs the whole expiry path. - macro_rules! repair_fatal { - ($($msg:tt)*) => {{ - let msg = format!($($msg)*); - error!(%msg, "cursor-expiry repair failed; aborting watch"); - handle.abort(); - return Err(KvError::WatchError(msg)); - }}; - } - loop { tokio::select! { biased; @@ -597,7 +315,7 @@ where // re-delivered on the next resume — and return the applied cursor. res = shutdown.changed() => { if res.is_err() || *shutdown.borrow() { - flush!(); + fold.flush().await.map_err(stop)?; handle.abort(); // Observe the task's terminal state. An abort surfaces as a // cancelled JoinError, which we ignore; a genuine panic that @@ -607,231 +325,27 @@ where { warn!(error = %join, "watch task panicked at shutdown"); } - return Ok(applied); + return Ok(fold.applied().clone()); } } // Batch window elapsed. - () = &mut sleep, if batch_deadline.is_some() => { - flush!(); + () = &mut sleep, if fold.is_open() => { + fold.flush().await.map_err(stop)?; } // Cursor-expiry repair. Placed before `rx.recv()` (biased), but - // ordering against the update stream comes from the drain below - // and the ack/reply protocol: the watch task is parked until this - // arm finishes, so nothing from its next watch can interleave. + // ordering against the update stream comes from the backlog drain + // in `repair::fold_in` and the ack/reply protocol: the watch task + // is parked until this arm finishes, so nothing from its next + // watch can interleave. req = async { repairs.as_mut().expect("arm guarded by is_some").recv().await }, if repairs.is_some() => { match req { Some(req) => { - // Fold everything the watch task delivered before it - // asked for this repair. A floor-guard trip ends a - // live watch mid-stream with updates still buffered in - // the channel: the repair must see them (it diffs the - // fold) and supersede them (one applied after the - // repair would resurrect pre-repair state, and could - // even move the cursor backward). The watch task is - // parked on the ack/reply, so this drains exactly that - // backlog. - while let Ok(u) = rx.try_recv() { - ingest!(u); - } - flush!(); - // Both repairs diff the STORE, so it must hold - // everything delivered. A transient store failure - // re-queues the batch in `raw_batch`, which the diff - // can't see: it would miss a re-queued put that a - // deletion during the gap has since removed, and the - // put would then commit and resurrect the key. Retry - // until the store has it all; a persistent failure - // hits the flush's own fail-stop bound. - while !raw_batch.is_empty() { - flush!(); - } - match req { - RepairRequest::Relist { live_keys, ack } => { - // Diff the fold's in-scope keys against the - // bucket's live listing; anything the fold - // holds that the bucket no longer does vanished - // during the gap (its delete marker evicted with - // the cursor), so synthesize the delete the - // re-list can't deliver. - let live: std::collections::HashSet<&str> = - live_keys.iter().map(String::as_str).collect(); - let mut stale: Vec = Vec::new(); - if let Some(st) = &store { - for prefix in &scope_prefixes { - // Stream the fold's keys rather than - // `range()`, which buffers every in-scope - // entry — values included — into one Vec. - // On an on-disk backend holding a fold - // larger than RAM (the case those backends - // exist for), an All-scope resync would - // materialize the entire fold on the - // repair path. Only the keys matter. - if let Err(e) = st.for_each_in_range(prefix, |entry| { - if !live.contains(entry.key.as_str()) { - stale.push(entry.key); - } - Ok(()) - }) { - // FATAL, not a degrade: an incomplete - // diff silently leaves deleted keys in - // the fold forever (tests/model.rs - // proves the divergence reachable - // under degrade semantics). Fail the - // watch; the restart re-runs the - // resume → expiry → resync from - // scratch. - warn!(error = %e, prefix = %prefix, - "resync fold scan failed; aborting watch rather than diverging"); - handle.abort(); - return Err(KvError::WatchError(format!( - "cursor-expired resync failed listing fold prefix {prefix:?}: {e}" - ))); - } - } - } - // Overlapping prefixes can list a key twice. - stale.sort_unstable(); - stale.dedup(); - if !stale.is_empty() { - warn!(stale = stale.len(), "cursor-expired resync: deleting keys that vanished during the gap"); - } - for key in stale { - // Synthetic: carries no revision (unknown - // version) and so never advances the cursor. - let u = KvUpdate::Delete { - key, - version: crate::kv::VersionToken::unknown(), - }; - if store.is_some() { - raw_batch.push(u.clone()); - } - if let Some(parsed) = parse(&u) { - batch.push(parsed); - } - } - flush!(); - // Ack AFTER the deletes are applied: the watch - // task is holding the fallback watch until it - // hears back, which is what orders deletes - // before the re-list - // (tests/model_resync_order.rs proves the - // barrier load-bearing). If the flush's STORE - // apply failed transiently, the deletes sit - // re-queued at the FRONT of the raw batch — - // still strictly before any re-list put in the - // eventual cumulative commit, and the domain - // apply saw them before this ack either way. - let _ = ack.send(()); - } - RepairRequest::Restore { - restored, - live, - reply, - } => { - let target = restored.manifest().cursor.clone(); - // The authoritative ahead check (the watch - // task's ran against its resume cursor; a - // mid-watch expiry delivered past that). A - // restore never moves the fold backward. - if cursor_rank(&target) <= cursor_rank(&applied) { - let msg = format!( - "cursor-expired restore: the latest artifact (cursor \ - {target:?}) is not ahead of the fold (cursor {applied:?}); \ - nothing newer to restore from" - ); - error!(%msg, "cursor-expiry repair refused"); - // The watch task fails with this; its - // channel closing ends the loop. - let _ = reply.send(Err(KvError::WatchError(msg))); - } else { - let Some(st) = store.take() else { - unreachable!("repairs are armed only with a store") - }; - let prefixes = scope_prefixes.clone(); - let (st, mut restored, diff) = - match tokio::task::spawn_blocking(move || { - let diff = - restore_diff(&st, restored.fold(), &prefixes, &live); - (st, restored, diff) - }) - .await - { - Ok(v) => v, - Err(e) => repair_fatal!( - "cursor-expired restore: diff task panicked: {e}" - ), - }; - store = Some(st); - let diff = match diff { - Ok(d) => d, - Err(e) => repair_fatal!( - "cursor-expired restore: diffing the fold against the \ - artifact failed: {e}" - ), - }; - info!( - from = ?applied, - to = ?target, - changed = diff.len(), - "cursor-expired restore: replacing the in-scope fold with the artifact's" - ); - // Fold the difference through parse/apply/ - // store in `max`-sized chunks, values read - // back per chunk. These carry no stream - // position: every chunk commits under the - // old cursor, so a crash part-way re-runs - // the (idempotent) restore on restart. - let mut ops = diff.into_iter(); - loop { - let chunk: Vec<_> = ops.by_ref().take(batch_cap).collect(); - if chunk.is_empty() { - break; - } - let (r, updates) = match tokio::task::spawn_blocking(move || { - let updates = materialize(restored.fold(), chunk); - (restored, updates) - }) - .await - { - Ok(v) => v, - Err(e) => repair_fatal!( - "cursor-expired restore: read task panicked: {e}" - ), - }; - restored = r; - let updates = match updates { - Ok(u) => u, - Err(e) => repair_fatal!( - "cursor-expired restore: reading the artifact failed: {e}" - ), - }; - for u in updates { - if store.is_some() { - raw_batch.push(u.clone()); - } - if let Some(parsed) = parse(&u) { - batch.push(parsed); - } - } - flush!(); - } - // Now the fold IS the artifact's (in - // scope): take its cursor, committed to - // the store with the final chunk's - // re-queued remainder, if any. - batch_high = target.clone(); - flush!(); - // Closing an LSM can block; drop the - // temporary fold and its scratch dir off - // the async thread. - let _ = tokio::task::spawn_blocking(move || drop(restored)).await; - let _ = reply.send(Ok(target)); - } - } - } + repair::fold_in(&mut fold, req, &mut rx, &prefixes) + .await + .map_err(stop)?; } None => repairs = None, } @@ -839,60 +353,11 @@ where // Export request. Placed after shutdown/window (they stay prompt) // and before `rx.recv()` so a firehose of updates cannot starve an - // export indefinitely. The pending batch is flushed first, so the - // exported cursor is exactly the applied cursor — except when - // that flush's store apply transiently failed (batch re-queued): - // the export then captures the store's OWN lagging cursor, which - // is still self-consistent with its contents (cursor authority, - // tests/model_applied.rs); the artifact never includes unfolded - // data, and a bootstrap from it simply replays the short gap. - // The export itself runs on a blocking task with the store moved - // in and taken back — the same offload the flush path uses. The - // artifact's manifest records this watch's scope, so a restore can - // tell whether it covers the restoring watcher. + // export indefinitely. req = async { exports.as_mut().expect("arm guarded by is_some").recv().await }, if exports.is_some() => { match req { - Some(ExportRequest { dest_dir, reply }) => { - flush!(); - match store.take() { - Some(mut st) => { - let scope = scope_prefixes.clone(); - match tokio::task::spawn_blocking(move || { - let res = st.export_to(&dest_dir).and_then(|_| { - crate::artifact::stamp_scope(&dest_dir, &scope) - }); - (st, res) - }) - .await - { - // Hand the store back on any clean return; an - // export failure goes to the requester only — - // the watch keeps running (the snapshot is a - // cache). A panicked task lost the store, - // which breaks the resume guarantee: fatal, - // same as the flush path's apply panic. - Ok((st, res)) => { - store = Some(st); - let _ = reply.send(res); - } - Err(e) => { - warn!(error = %e, "snapshot export task panicked; aborting watch"); - handle.abort(); - return Err(KvError::WatchError(format!( - "snapshot export task panicked: {e}" - ))); - } - } - } - None => { - let _ = reply.send(Err(SnapshotError::Backend( - "watch_applied runs without a snapshot store; nothing to export" - .into(), - ))); - } - } - } + Some(req) => fold.export(req, &prefixes).await.map_err(stop)?, // Sender dropped: disarm the arm for the rest of the run. None => exports = None, } @@ -900,14 +365,25 @@ where update = rx.recv() => { match update { - Some(u) => ingest!(u), + Some(u) => { + // Arm the window on the first received update of a + // batch — even a parse-rejected one, so the cursor + // advances within `window` even through a run of + // irrelevant keys. + if !fold.is_open() { + sleep.as_mut().reset(tokio::time::Instant::now() + config.window); + } + if fold.ingest(u) { + fold.flush().await.map_err(stop)?; + } + } None => { // Stream closed. Flush the remainder, then surface the // watch task's terminal result: a clean end returns the // applied cursor, an error propagates. - flush!(); + fold.flush().await.map_err(stop)?; return match handle.await { - Ok(Ok(())) => Ok(applied), + Ok(Ok(())) => Ok(fold.applied().clone()), Ok(Err(e)) => Err(e), Err(join) => Err(KvError::WatchError(format!( "watch task panicked: {join}" @@ -920,362 +396,324 @@ where } } -/// Does the fold hold any entry? Stops at the first. -fn fold_has_entries(store: &S) -> Result { - let mut found = false; - match store.for_each_in_range("", |_| { - found = true; - // Stop the scan; `found` carries the answer. - Err(SnapshotError::Backend(String::new())) - }) { - Err(e) if !found => Err(e), - _ => Ok(found), - } +/// Consecutive store-apply failures that fail-stop the watch. A transient +/// failure re-queues its raw batch (cursor authority: the store's cursor and +/// contents advance together, always); a persistent streak fail-stops before +/// the re-queued backlog grows without bound. +const MAX_STORE_APPLY_FAILURES: u32 = 16; + +/// The cursor-after-apply fold behind [`watch_applied`]: the pending batch, +/// the applied cursor, and the store they commit to. Every change to the +/// fold — stream updates, repair corrections, exports — goes through it, so +/// the invariant in the module docs is enforced in one place: the cursor +/// advances only in [`flush`](Self::flush), after `apply` returns. +pub(crate) struct Fold { + store: Option, + parse: P, + apply: A, + on_applied: O, + // Parsed updates for `apply`: the consumer's domain view. Pre-sized to the + // flush bound so no batch ever re-climbs the reallocation ladder. + batch: Vec, + batch_cap: usize, + // Flush when either batch reaches this many updates. + max: usize, + // Raw received updates for the durable store, in revision order. Only + // populated when a store is present; the store folds the *raw* updates + // (including ones `parse` rejected — they are still part of the bucket's + // state). + raw: Vec, + // The version of the most recently *received* update since the last + // flush — including updates `parse` rejected. NATS delivers in revision + // order, so the last received is the highest, and advancing the cursor to + // it after a single atomic `apply` is correct: having seen the max means + // we've seen everything below it, and a rejected entry is still "nothing + // to apply", hence covered. Reset to `none()` after every flush. + batch_high: WatchCursor, + applied: WatchCursor, + store_fail_streak: u32, + // A batch opened since the last flush (the window timer is armed). + open: bool, } -/// Run the underlying watch for `scope`, resuming from `resume` when it carries -/// a position, with cursor-expiry repair: an expiry (at resume, or mid-watch -/// from the floor guard) runs the planned repair and then either falls back to -/// the full-scope re-list (Relist / nothing armed) or resumes from the restored -/// artifact's cursor (Restore) — which can itself expire later and go round -/// again, each time only for a strictly newer artifact. -/// -/// `unanchored`: the fold holds data but no cursor (a torn first checkpoint, -/// or a populated store started without one). A re-list alone never removes -/// what it doesn't deliver, so that start is repaired like an expiry at -/// revision 0 before watching. -async fn run_watch( - watcher: &dyn KvWatcher, - scope: &WatchScope, - resume: Option, - unanchored: bool, - repair: Option>, - tx: mpsc::Sender, -) -> Result<(), KvError> { - // Resume only when the cursor carries a real position; an absent or `none()` - // cursor falls through to a full watch. Binding `cursor` here makes "we have a - // resume position" structural — there is no separate bool whose truth a later - // edit could let drift from the `Some`. - let mut resume = resume.filter(|c| !c.is_none()); - - // A cursor-less start that must be repaired before it watches: a fold - // with data and no cursor (any repair), or an empty fold on a bucket that - // already evicted current values with a restore armed (its re-list would - // seed an incomplete fold). - let mut repair_first = resume.is_none() - && match &repair { - // (A handle exists only for a real repair mode.) - Some(_) if unanchored => { - warn!( - "the fold holds data but no cursor; repairing it like an expired cursor \ - before watching (a re-list alone never removes what it doesn't deliver)" - ); - true - } - Some(h) => { - let seed = fresh_start_needs_restore(watcher, h).await?; - if seed { - warn!( - "no resume cursor, and the bucket has already evicted current values: a \ - re-list would seed an incomplete fold; restoring from the latest artifact" - ); - } - seed - } - None => { - // Nothing armed: the re-list stands as the fold. Say so when - // the bucket has already evicted current values — the copy - // then lacks whatever aged out, and nothing will repair it. - if watcher - .retention() - .await? - .is_some_and(|r| !listing_is_truth(&r)) - { - warn!( - "no resume cursor and no repair armed, on a bucket that has already \ - evicted current values: the re-list lacks whatever aged out. Wire a \ - store and ExpiryRepair::Auto to seed from an artifact" - ); - } - false - } - }; - - loop { - let cursor = if std::mem::take(&mut repair_first) { - WatchCursor::none() - } else { - let Some(cursor) = resume.take() else { - return watch_scope(watcher, scope, tx).await; - }; - match watch_scope_from(watcher, scope, &cursor, tx.clone()).await { - Err(KvError::CursorExpired) => cursor, - other => return other, - } - }; - match plan_repair(watcher, repair.as_ref()).await? { - Plan::ReListOnly => { - warn!( - "watch cursor expired with no repair armed (needs a store and a reader or \ - restore source); falling back to the re-list alone — keys deleted during \ - the gap may persist in the fold" - ); - return watch_scope(watcher, scope, tx).await; - } - Plan::Relist(reader, repairs) => { - warn!( - "watch cursor expired; resyncing stale keys, then falling back to the full re-list" - ); - resync_stale_keys(scope, reader, repairs).await?; - return watch_scope(watcher, scope, tx).await; - } - Plan::Restore(reader, source, repairs) => { - warn!("watch cursor expired; restoring the fold from the latest artifact"); - let restored = - restore_from_artifact(watcher, scope, reader, source, repairs, &cursor) - .await - .map_err(|e| match e { - KvError::WatchError(msg) if cursor.is_none() => { - KvError::WatchError(format!( - "{msg}. This node has no cursor, so it can only seed from an \ - artifact. If none exists yet (first deploy onto a bucket that \ - already evicted values), start one node with ExpiryRepair::None \ - — accepting that values which already aged out are gone — and \ - publish an export from it" - )) - } - other => other, - })?; - resume = Some(restored); - } +impl Fold +where + U: Send, + S: SnapshotStore + Send + 'static, + P: FnMut(&KvUpdate) -> Option + Send, + A: FnMut(Vec) + Send, + O: FnMut(WatchCursor) + Send, +{ + fn new( + store: Option, + parse: P, + apply: A, + on_applied: O, + max: usize, + applied: WatchCursor, + ) -> Self { + // `max(1)` only guards a nonsensical `max = 0` config. + let batch_cap = max.max(1); + Self { + store, + parse, + apply, + on_applied, + batch: Vec::with_capacity(batch_cap), + batch_cap, + max, + raw: Vec::new(), + batch_high: WatchCursor::none(), + applied, + store_fail_streak: 0, + open: false, } } -} -/// The full-scope (state-sync re-list) watch for `scope`. -async fn watch_scope( - watcher: &dyn KvWatcher, - scope: &WatchScope, - tx: mpsc::Sender, -) -> Result<(), KvError> { - match scope { - WatchScope::All => watcher.watch_all(tx).await, - WatchScope::Prefix(prefix) => watcher.watch_prefix(prefix, tx).await, - WatchScope::Prefixes(prefixes) => { - let refs: Vec<&str> = prefixes.iter().map(String::as_str).collect(); - watcher.watch_prefixes(&refs, tx).await + /// Take one update off the watch stream into the pending batch. Returns + /// `true` when a batch is full and must be flushed: the parsed batch, or — + /// when persisting — the raw batch, so a window packed with + /// parse-rejected updates can't grow it without bound. + pub(crate) fn ingest(&mut self, u: KvUpdate) -> bool { + // Cursor authority: every received update bumps the pending + // high-water, regardless of whether `parse` keeps it — but only when + // it carries a real position. An unknown version (e.g. an unparseable + // ACK subject on the hand-built multi-prefix consumer path) must + // neither mint a fake cursor nor clobber the real high from earlier in + // the batch; skipping it under-advances at worst, and re-delivery on + // resume is idempotent. + if !u.version().is_unknown() { + self.batch_high = WatchCursor::from_version(u.version().clone()); } - } -} - -/// The delta watch for `scope`, resuming after `cursor`. -async fn watch_scope_from( - watcher: &dyn KvWatcher, - scope: &WatchScope, - cursor: &WatchCursor, - tx: mpsc::Sender, -) -> Result<(), KvError> { - match scope { - WatchScope::All => watcher.watch_all_from(cursor, tx).await, - WatchScope::Prefix(prefix) => watcher.watch_prefix_from(prefix, cursor, tx).await, - WatchScope::Prefixes(prefixes) => { - let refs: Vec<&str> = prefixes.iter().map(String::as_str).collect(); - watcher.watch_prefixes_from(&refs, cursor, tx).await + self.correct(u); + self.open = true; + self.batch.len() >= self.max || self.raw.len() >= self.max + } + + /// Queue a repair's state correction (a synthetic delete, a restored + /// entry). It runs through `parse`/`apply`/store like a stream update, + /// but carries no stream position, so it never moves the cursor and + /// doesn't open the batch window: the repair flushes it itself. + pub(crate) fn correct(&mut self, u: KvUpdate) { + let parsed = (self.parse)(&u); + // Only when a store is present, so the no-persistence path keeps its + // zero-copy cost. + if self.store.is_some() { + self.raw.push(u); + } + if let Some(parsed) = parsed { + self.batch.push(parsed); } } -} -/// Is the bucket's key listing the truth — does "not listed" mean "deleted"? -/// Yes when retention never evicts current values, and also when it can but -/// has never evicted anything (the first retained revision is still 1). Read -/// live; `None` when the backend can't say. -fn listing_is_truth(r: &Retention) -> bool { - !r.evicts_current_values || resume_window_ok(0, r.first_revision) -} - -/// Decide how to repair one expiry, reading the bucket's retention live when -/// the choice depends on it. -async fn plan_repair<'a, S>( - watcher: &dyn KvWatcher, - repair: Option<&'a RepairHandle>, -) -> Result, KvError> { - let Some(h) = repair else { - return Ok(Plan::ReListOnly); - }; - Ok(match &h.mode { - ExpiryRepair::None => Plan::ReListOnly, - ExpiryRepair::Relist(reader) => { - if watcher - .retention() - .await? - .is_some_and(|r| !listing_is_truth(&r)) + /// Set the pending cursor as if the stream had delivered `cursor`: a + /// restore's last step, taking the artifact's cursor. The next flush + /// commits it. + pub(crate) fn advance_to(&mut self, cursor: WatchCursor) { + self.batch_high = cursor; + } + + /// Flush the pending batch, in order: run the domain `apply` (if + /// non-empty) to completion, advance the cursor, fold the raw batch + + /// cursor durably into the store, then fire `on_applied`. The store fold + /// runs on a blocking task (its `apply` may block on I/O), moving the + /// store in and taking it back. A TRANSIENT store error re-queues the raw + /// batch for cumulative commit on the next flush (the watch continues; the + /// store's cursor never advances past data it dropped) and a persistent + /// failure streak is fatal; a panicked blocking task drops the store + /// irrecoverably, which breaks the resume-after-restart guarantee, so it + /// is surfaced as fatal. + pub(crate) async fn flush(&mut self) -> Result<(), KvError> { + self.open = false; + // Nothing received since the last flush → nothing to do at all. + // (`raw` can be non-empty with no cursor advance only via a repair's + // corrections, or a re-queued batch.) + if self.batch.is_empty() && self.raw.is_empty() && self.batch_high.is_none() { + return Ok(()); + } + if !self.batch.is_empty() { + // INVARIANT: apply() runs and RETURNS before any cursor advance + // below. Move the batch out so a panicking apply can't leave + // half-consumed state behind. + // + // `replace` (not `take`) leaves a pre-sized Vec behind so each + // batch after the first doesn't re-climb the reallocation ladder + // (4→8→…→cap). + (self.apply)(std::mem::replace( + &mut self.batch, + Vec::with_capacity(self.batch_cap), + )); + } + let advanced = !self.batch_high.is_none(); + if advanced { + self.applied = self.batch_high.clone(); + } + // A cursor-only advance (nothing raw to fold) still commits to the + // store: a restore's final step moves the cursor to the artifact's + // with no updates left. In the steady state every advance carries raw + // updates, so this changes nothing there. + if (!self.raw.is_empty() || advanced) + && let Some(mut st) = self.store.take() + { + let raw = std::mem::take(&mut self.raw); + // Fold at the post-advance cursor. A repair-only batch (synthetic + // updates) leaves the cursor where it was (they are a state + // correction, not log entries), which is safe: an unchanged — + // possibly expired — cursor only ever re-runs the same repair on + // the next restart. + let cur = self.applied.clone(); + // Hand the store AND the raw batch back on a clean return: a + // *failed* apply (Ok(Err)) RE-QUEUES the batch so the next flush + // commits it cumulatively — the store's cursor and contents always + // advance together. Dropping the failed batch instead lets the + // NEXT successful flush advance the cursor over a hole that + // survives every restart (reproduced by + // `transient_store_failure_never_leaves_a_cursor_gap`). Only a + // *panicked* task (Err) loses the store: fatal. + match tokio::task::spawn_blocking(move || { + let res = st.apply(&raw, &cur); + (st, raw, res) + }) + .await { - let msg = "watch cursor expired on a bucket whose retention evicts current \ - values (max_age, per-message TTL, or discard:old): its key listing \ - can't tell a deleted key from an aged-out one, so the key-listing \ - resync would delete valid keys. Refusing. Repair from artifacts with \ - ExpiryRepair::Restore or ExpiryRepair::Auto (or accept a re-list-only \ - fallback with ExpiryRepair::None)"; - error!(msg); - return Err(KvError::WatchError(msg.into())); + Ok((st, _raw, Ok(()))) => { + self.store = Some(st); + self.store_fail_streak = 0; + } + Ok((st, raw, Err(e))) => { + self.store_fail_streak += 1; + let streak = self.store_fail_streak; + if streak >= MAX_STORE_APPLY_FAILURES { + // A persistently failing store would otherwise grow + // the re-queued batch without bound while the fold + // silently stales. Fail-stop: the restart refolds the + // tail from the store's last good cursor. + warn!(error = %e, streak, + "snapshot store apply failing persistently; aborting watch"); + return Err(KvError::WatchError(format!( + "snapshot store apply failed {streak} consecutive times: {e}" + ))); + } + warn!(error = %e, streak, + "snapshot store apply failed; batch re-queued for the next flush"); + self.store = Some(st); + // Prepend: the failed range precedes anything received + // since (stream order is preserved for the eventual + // cumulative commit). + let newer = std::mem::replace(&mut self.raw, raw); + self.raw.extend(newer); + } + Err(e) => { + warn!(error = %e, "snapshot store task panicked; aborting watch"); + return Err(KvError::WatchError(format!( + "snapshot store task panicked: {e}" + ))); + } } - Plan::Relist(reader, &h.tx) } - ExpiryRepair::Restore { reader, restore } => Plan::Restore(reader, restore, &h.tx), - ExpiryRepair::Auto { reader, restore } => match watcher.retention().await? { - Some(r) if listing_is_truth(&r) => Plan::Relist(reader, &h.tx), - _ => Plan::Restore(reader, restore, &h.tx), - }, - }) -} - -/// Must an EMPTY cursor-less start seed from an artifact? Only with a restore -/// armed, on a bucket whose re-list is no longer complete (it evicted current -/// values). A bucket whose listing is the truth re-lists completely. -async fn fresh_start_needs_restore( - watcher: &dyn KvWatcher, - h: &RepairHandle, -) -> Result { - if !matches!( - h.mode, - ExpiryRepair::Restore { .. } | ExpiryRepair::Auto { .. } - ) { - return Ok(false); - } - Ok(watcher - .retention() - .await? - .is_some_and(|r| !listing_is_truth(&r))) -} - -/// The artifact restore, watch-task half: check the newest artifact before -/// downloading it (ahead of `local`, covers the scope, still inside -/// retention), download it, check what actually arrived, then hand it to the -/// main loop and wait for the cursor it restored to. -async fn restore_from_artifact( - watcher: &dyn KvWatcher, - scope: &WatchScope, - reader: &Arc, - source: &Arc>, - repairs: &mpsc::Sender>, - local: &WatchCursor, -) -> Result { - let prefixes = scope.prefixes(); - let fail = |msg: String| { - error!(%msg, "cursor-expiry restore refused; no safe recovery"); - KvError::WatchError(format!("cursor-expired restore: {msg}")) - }; - - let first = watcher.retention().await?.map(|r| r.first_revision); - let latest = source - .latest() - .await - .map_err(|e| fail(format!("reading the latest artifact failed: {e}")))?; - check_restore(&latest, local, first, &prefixes).map_err(fail)?; - - let restored = source - .fetch() - .await - .map_err(|e| fail(format!("fetching the latest artifact failed: {e}")))?; - // The pointer may have advanced between the peek and the download, and - // retention may have moved: check what actually arrived, against now. - let first = watcher.retention().await?.map(|r| r.first_revision); - check_restore(restored.manifest(), local, first, &prefixes).map_err(fail)?; - let age_secs = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|now| { - now.as_secs() - .saturating_sub(restored.manifest().created_at_unix) - }) - .unwrap_or(0); - info!( - cursor = ?restored.manifest().cursor, - age_secs, - first_retained = ?first, - "restoring the fold from the latest artifact" - ); - - // The bucket's live keys, listed after the artifact's cursor is fixed: - // the restore never deletes one of these (see `restore_diff`). A failed - // listing fails the watch, like the key-listing repair's. - let mut live = std::collections::HashSet::new(); - for prefix in &prefixes { - let keys = reader - .keys(prefix) - .await - .map_err(|e| fail(format!("listing live keys under {prefix:?} failed: {e}")))?; - live.extend(keys); + if advanced { + (self.on_applied)(self.applied.clone()); + self.batch_high = WatchCursor::none(); + } + Ok(()) } - let (reply_tx, reply_rx) = oneshot::channel(); - repairs - .send(RepairRequest::Restore { - restored, - live, - reply: reply_tx, + /// Flush until the store holds every update received: a transient store + /// failure re-queues the batch, so retry; a persistent one hits the + /// flush's own fail-stop bound. + pub(crate) async fn settle(&mut self) -> Result<(), KvError> { + self.flush().await?; + while !self.raw.is_empty() { + self.flush().await?; + } + Ok(()) + } + + /// Export the fold (see [`ExportRequest`]). The pending batch is flushed + /// first, so the exported cursor is exactly the applied cursor — except + /// when that flush's store apply transiently failed (batch re-queued): + /// the export then captures the store's OWN lagging cursor, which is still + /// self-consistent with its contents (cursor authority, + /// tests/model_applied.rs); the artifact never includes unfolded data, and + /// a bootstrap from it simply replays the short gap. The export runs on a + /// blocking task with the store moved in and taken back, and the + /// manifest records the watch's `scope`, so a restore can tell whether it + /// covers the restoring watcher. + /// + /// An export failure goes to the requester only — the watch keeps + /// running. A panicked task lost the store, which breaks the resume + /// guarantee: fatal, same as the flush path's. + pub(crate) async fn export( + &mut self, + ExportRequest { dest_dir, reply }: ExportRequest, + scope: &[String], + ) -> Result<(), KvError> { + self.flush().await?; + let Some(mut st) = self.store.take() else { + let _ = reply.send(Err(SnapshotError::Backend( + "watch_applied runs without a snapshot store; nothing to export".into(), + ))); + return Ok(()); + }; + let scope = scope.to_vec(); + match tokio::task::spawn_blocking(move || { + let res = st + .export_to(&dest_dir) + .and_then(|_| crate::artifact::stamp_scope(&dest_dir, &scope)); + (st, res) }) .await - .map_err(|_| KvError::WatchError("watch loop ended during a restore".into()))?; - reply_rx - .await - .map_err(|_| KvError::WatchError("watch loop dropped a restore reply".into()))? -} - -/// Cursor-expired stale-key resync (the key-listing diff), run BEFORE the -/// fallback watch is established: list the scope's live keys, hand them to the -/// main loop (which diffs them against the fold and applies synthetic -/// deletes), and wait for the ack. That ordering — deletes applied, then -/// fallback watch armed — is what makes a delete-then-recreate during the gap -/// converge: the synthetic delete always lands before the re-list put. -/// -/// Sound only on a bucket that never evicts current values; `plan_repair` -/// refuses it otherwise. -/// -/// A FAILED listing is **fatal** — it fails the watch rather than degrading. -/// The resync is load-bearing for the "stale, never corrupt" convergence -/// guarantee: a silently degraded resync leaves the fold holding keys the -/// bucket deleted, with one warn line as the only witness (`tests/model.rs` -/// proves this divergence reachable under degrade semantics). Failing the -/// watch turns the violated guarantee into a visible error; the caller's -/// restart re-resumes, hits `CursorExpired` again, and retries the resync from -/// scratch. -async fn resync_stale_keys( - scope: &WatchScope, - reader: &Arc, - repairs: &mpsc::Sender>, -) -> Result<(), KvError> { - let mut live_keys = Vec::new(); - for prefix in scope.prefixes() { - match reader.keys(&prefix).await { - Ok(keys) => live_keys.extend(keys), + { + Ok((st, res)) => { + self.store = Some(st); + let _ = reply.send(res); + Ok(()) + } Err(e) => { - return Err(KvError::WatchError(format!( - "cursor-expired resync failed listing live keys under {prefix:?}: {e}; \ - failing the watch rather than silently keeping stale keys" - ))); + warn!(error = %e, "snapshot export task panicked; aborting watch"); + Err(KvError::WatchError(format!( + "snapshot export task panicked: {e}" + ))) } } } - let (ack_tx, ack_rx) = oneshot::channel(); - if repairs - .send(RepairRequest::Relist { - live_keys, - ack: ack_tx, - }) - .await - .is_ok() - { - // A dropped ack (main loop shutting down) just means the fallback watch - // is about to die with it; nothing to recover. - let _ = ack_rx.await; + + /// The applied cursor: everything at or below it has been applied. + pub(crate) fn applied(&self) -> &WatchCursor { + &self.applied + } + + /// The flush bound, for a repair's own chunking. + pub(crate) fn batch_cap(&self) -> usize { + self.batch_cap + } + + /// A batch is open: the window timer must be armed. + fn is_open(&self) -> bool { + self.open + } + + /// The store, between flushes. + pub(crate) fn store(&self) -> Option<&S> { + self.store.as_ref() + } + + /// Move the store out (to a blocking task); [`put_store`](Self::put_store) + /// hands it back. A flush in between would skip the store fold. + pub(crate) fn take_store(&mut self) -> Option { + self.store.take() + } + + /// Hand back a store taken with [`take_store`](Self::take_store). + pub(crate) fn put_store(&mut self, store: S) { + self.store = Some(store); } - Ok(()) } #[cfg(test)] mod tests { use super::*; - use crate::kv::{KvEntry, VersionToken}; + use crate::kv::{KvEntry, KvReader, VersionToken}; use crate::snapshot::AppendLogSnapshot; use async_trait::async_trait; use std::sync::Mutex; diff --git a/src/kv.rs b/src/kv.rs index 8ed0dc5..2e4513b 100644 --- a/src/kv.rs +++ b/src/kv.rs @@ -41,6 +41,15 @@ impl WatchCursor { pub(crate) fn version(&self) -> &VersionToken { &self.0 } + + /// The cursor's position for ordering — the artifact pointer swap, the + /// restore guard: its u64 revision, or 0 when it has none. Those + /// protocols need a backend with u64 positions; a revisionless cursor + /// ranks lowest, so it never supersedes a real one and is never "ahead". + /// The ONE place cursors are ranked. + pub(crate) fn rank(&self) -> u64 { + self.as_u64().unwrap_or(0) + } } /// Error type for KV operations. diff --git a/src/nats.rs b/src/nats.rs index 5c0760f..a81348a 100644 --- a/src/nats.rs +++ b/src/nats.rs @@ -113,7 +113,7 @@ async fn build_connect_options( /// Connect to NATS with various authentication methods. /// -/// Supports the auth-priority order documented on [`build_connect_options`]. +/// Supports the auth-priority order documented on `build_connect_options`. /// This is the standalone helper; the [`Connection`] impl builds options the /// same way but also installs a health-tracking event callback. pub async fn nats_connect( @@ -1298,7 +1298,13 @@ impl NatsKvWatcher { /// creation is the same exposure any live consumer has against /// aggressive retention; the check bounds the silent gap to that /// milliseconds-scale window, where the prior behavior left it unbounded. - async fn check_resume_window(&self, revision: u64) -> Result<(), KvError> { + /// + /// Returns the KV stream handle it read, for a caller that creates its + /// consumer on it. + async fn check_resume_window( + &self, + revision: u64, + ) -> Result { let stream = timed(self.js.get_stream(format!("KV_{}", self.bucket))) .await? .map_err(|e| { @@ -1315,7 +1321,42 @@ impl NatsKvWatcher { ); return Err(KvError::CursorExpired); } - Ok(()) + Ok(stream) + } + + /// Create a resume consumer (or watch) for `revision`, the ONE way every + /// `*_from` path does it: the window checked before creation + /// ([`check_resume_window`](Self::check_resume_window); `create` gets the + /// stream handle it read), a creation error that names the sequence + /// problem classified as [`KvError::CursorExpired`] (servers that do + /// error on a compacted start), and the window re-checked AFTER the + /// consumer exists — head eviction between the check and the creation + /// would otherwise clamp silently. `scope` is for the log line. + async fn create_resume( + &self, + revision: u64, + scope: &(dyn std::fmt::Debug + Sync), + create: F, + ) -> Result + where + E: std::fmt::Display, + F: FnOnce(async_nats::jetstream::stream::Stream) -> Fut, + Fut: std::future::Future>, + { + let stream = self.check_resume_window(revision).await?; + let created = match timed(create(stream)).await? { + Ok(c) => c, + Err(e) => { + let err = e.to_string(); + if is_cursor_expired_error(&err) { + warn!(revision, ?scope, error = %err, "cursor expired at consumer creation"); + return Err(KvError::CursorExpired); + } + return Err(KvError::WatchError(err)); + } + }; + self.check_resume_window(revision).await?; + Ok(created) } } @@ -1398,24 +1439,10 @@ impl KvWatcher for NatsKvWatcher { Some(rev) if rev > 0 => rev, _ => return self.watch_all(tx).await, }; - self.check_resume_window(revision).await?; - let start = resume_start(revision)?; - let watcher = match timed(self.kv.watch_all_from_revision(start)).await? { - Ok(w) => w, - Err(e) => { - let err_str = e.to_string(); - if is_cursor_expired_error(&err_str) { - warn!(revision, error = %err_str, "cursor expired, caller should fall back to full watch"); - return Err(KvError::CursorExpired); - } - return Err(KvError::WatchError(err_str)); - } - }; - // Re-check AFTER the consumer exists: head eviction in the window - // between the pre-flight check and consumer creation would otherwise - // clamp silently. - self.check_resume_window(revision).await?; + let watcher = self + .create_resume(revision, &"all", |_| self.kv.watch_all_from_revision(start)) + .await?; info!(revision, "resumed watch from cursor"); // The LIVE floor guard takes over from here: in-band gapped-delivery @@ -1436,24 +1463,13 @@ impl KvWatcher for NatsKvWatcher { Some(rev) if rev > 0 => rev, _ => return self.watch_prefix(prefix, tx).await, }; - self.check_resume_window(revision).await?; - let nats_key = format!("{prefix}>"); let start = resume_start(revision)?; - let watcher = match timed(self.kv.watch_from_revision(&nats_key, start)).await? { - Ok(w) => w, - Err(e) => { - let err_str = e.to_string(); - if is_cursor_expired_error(&err_str) { - warn!(revision, prefix, error = %err_str, "cursor expired for prefix watch, caller should fall back"); - return Err(KvError::CursorExpired); - } - return Err(KvError::WatchError(err_str)); - } - }; - // Same post-create re-check as watch_all_from: close the - // check→create eviction window. - self.check_resume_window(revision).await?; + let watcher = self + .create_resume(revision, &prefix, |_| { + self.kv.watch_from_revision(&nats_key, start) + }) + .await?; info!(revision, prefix, "resumed prefix watch from cursor"); stream_watch(watcher, &tx).await @@ -1491,55 +1507,20 @@ impl KvWatcher for NatsKvWatcher { .map(|p| format!("{kv_prefix}{p}>")) .collect(); - let stream = timed(self.js.get_stream(format!("KV_{bucket}"))) - .await? - .map_err(|e| KvError::WatchError(format!("get KV stream: {e}")))?; - - // Same proactive expiry detection as `check_resume_window` (NATS - // silently clamps a below-head ByStartSequence; see that method's - // docs) — checked on the stream handle this path already fetched, - // via the shared protocol kernel. - let first = stream.cached_info().state.first_sequence; - if !crate::protocol::resume_window_ok(revision, first) { - warn!( - revision, - first_sequence = first, - ?prefixes, - "resume cursor is below the stream's first retained sequence; cursor expired" - ); - return Err(KvError::CursorExpired); - } - - let consumer = match timed(stream.create_consumer(push::OrderedConfig { + let start_sequence = resume_start(revision)?; + let config = push::OrderedConfig { deliver_subject: self.client.new_inbox(), description: Some("kv multi-prefix resume consumer".to_string()), filter_subjects, replay_policy: ReplayPolicy::Instant, - deliver_policy: DeliverPolicy::ByStartSequence { - start_sequence: resume_start(revision)?, - }, + deliver_policy: DeliverPolicy::ByStartSequence { start_sequence }, ..Default::default() - })) - .await? - { - Ok(c) => c, - Err(e) => { - // Same expiry classification as watch_all_from: a start sequence - // the stream has compacted past surfaces as a consumer-create - // error whose message names the sequence problem. - let err_str = e.to_string(); - if is_cursor_expired_error(&err_str) { - warn!(revision, ?prefixes, error = %err_str, "cursor expired for multi-prefix watch, caller should fall back"); - return Err(KvError::CursorExpired); - } - return Err(KvError::WatchError(err_str)); - } }; - - // Re-check AFTER the consumer exists (fresh stream info, not the - // handle's cached copy): closes the check→create eviction window, - // same as the single-filter resume paths. - self.check_resume_window(revision).await?; + let consumer = self + .create_resume(revision, &prefixes, |stream| async move { + stream.create_consumer(config).await + }) + .await?; let mut messages = timed(consumer.messages()) .await? @@ -1992,67 +1973,23 @@ mod tests { } } +/// The integration tests' `nats-server` harness, shared rather than copied. +#[cfg(test)] +#[path = "../tests/common/nats.rs"] +mod test_nats; + /// Live-server conformance tests for the floor guard /// ([`stream_watch_floor_guarded`]) — these drive the guarded loop DIRECTLY /// with a deliberately clamped `Watch`, which reproduces exactly the state /// retention leaves behind when it overruns a live consumer (the watcher /// methods' resume-time checks can't be raced deterministically from /// outside, but the guarded loop neither knows nor cares how its watch got -/// clamped). Spawns a throwaway `nats-server` (mise-installed, same pattern -/// as tests/common). +/// clamped). Spawns a throwaway `nats-server` (the shared `tests/common` +/// harness). #[cfg(test)] mod floor_guard_tests { + use super::test_nats::TestNats; use super::*; - use std::process::{Child, Command, Stdio}; - - struct TestServer { - child: Child, - url: String, - _dir: tempfile::TempDir, - } - - impl Drop for TestServer { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } - } - - async fn start_server() -> TestServer { - let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".into()); - let port = std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); - let dir = tempfile::tempdir().unwrap(); - let child = Command::new(&bin) - .args([ - "--jetstream", - "--addr", - "127.0.0.1", - "--port", - &port.to_string(), - "--store_dir", - dir.path().to_str().unwrap(), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap_or_else(|e| panic!("spawn {bin}: {e}; run `mise install`")); - let server = TestServer { - child, - url: format!("nats://127.0.0.1:{port}"), - _dir: dir, - }; - for _ in 0..100 { - if async_nats::connect(&server.url).await.is_ok() { - return server; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!("nats-server never became ready"); - } /// `(js, kv store)` with five revisions across five subjects (history 1). async fn seeded_bucket( @@ -2085,7 +2022,7 @@ mod floor_guard_tests { async fn purge_reclaims_bytes() { use crate::kv::KvPurge; - let server = start_server().await; + let server = TestNats::start().await; let client = async_nats::connect(&server.url).await.unwrap(); let js = async_nats::jetstream::new(client); let kv = js @@ -2154,7 +2091,7 @@ mod floor_guard_tests { /// `GuardRepair`-only-progress gate. #[tokio::test(flavor = "multi_thread")] async fn gapped_delivery_with_advanced_floor_trips() { - let server = start_server().await; + let server = TestNats::start().await; let (js, kv) = seeded_bucket(&server.url).await; // Evict revisions 1-3 outright: first_sequence becomes 4. @@ -2196,7 +2133,7 @@ mod floor_guard_tests { /// resume; this pins the discrimination explicitly.) #[tokio::test(flavor = "multi_thread")] async fn benign_interior_gap_passes() { - let server = start_server().await; + let server = TestNats::start().await; let (js, kv) = seeded_bucket(&server.url).await; // Overwrite k2 and k3: revisions 2 and 3 are interior-evicted @@ -2232,7 +2169,7 @@ mod floor_guard_tests { /// bucket does not. `first_revision` tracks the stream's head. #[tokio::test(flavor = "multi_thread")] async fn retention_reports_current_value_eviction() { - let server = start_server().await; + let server = TestNats::start().await; let conn = crate::NatsConnection::new(crate::NatsConnectionConfig { url: server.url.clone(), creds: None, diff --git a/src/protocol.rs b/src/protocol.rs index e225632..05465f5 100644 --- a/src/protocol.rs +++ b/src/protocol.rs @@ -1,24 +1,27 @@ -//! Protocol decision kernels — the load-bearing guards of the snapshot -//! export/import protocol, extracted as pure functions so the PRODUCTION -//! code and the exhaustive model checker (`tests/model.rs`) execute the -//! **same logic**, not two hand-synchronized copies of it. +//! Protocol decision kernels — every decision the snapshot export/import +//! protocol and the cursor-expiry repair rest on, extracted as pure functions +//! so the PRODUCTION code and the exhaustive model checkers (`tests/model*.rs`) +//! execute the **same logic**, not hand-synchronized copies of it. //! -//! Every function here is a guard whose correctness the machine-checked -//! theorems depend on. The call sites: -//! -//! | kernel | production | model | +//! | kernel | production | models | //! |----------------------------|-------------------------------------|--------------------| -//! | [`pointer_publish_allowed`]| `transport::swap_pointer` | `Act::Publish` | -//! | [`payload_prunable`] | `transport::ObjectStoreTransport::prune` | `Act::Prune` | -//! | [`resume_window_ok`] | `nats` resume paths (`check_resume_window`) | `Act::Resume` | -//! | [`restore_allowed`] | `applied` artifact restore (`check_restore`) | `Act::RestoreRead`, live `GuardRepair` | +//! | [`pointer_publish_allowed`]| `transport::swap_pointer` | `model.rs` `Act::Publish`, fleet `Publish` | +//! | [`payload_prunable`] | `transport::ObjectStoreTransport::prune` | `model.rs` `Act::Prune` | +//! | [`resume_window_ok`] | `nats` resume paths (`check_resume_window`), the live floor guard | every model | +//! | [`restore_allowed`] | `repair::check_restore` | `model.rs` `Act::RestoreRead`, live `GuardRepair`, repair-steps `WPlan`, fleet `Repair` | +//! | [`restore_ahead`] | `repair::check_restore`, the main loop's re-check (`repair::fold_in`) | repair-steps restore reply | +//! | [`listing_is_truth`] | `repair` (plan, cursor-less start), `Snapshot::stale_keys` | every model's view of retention | +//! | [`plan_repair`] | `repair::plan`, `Snapshot::stale_keys` | expiry dispatch in every model | +//! | [`cursorless_start_needs_repair`] | `repair::run_watch` | repair-steps `WStart`, fleet `Start` | +//! | [`restore_key`] | `repair::restore_diff` | repair-steps restore op | //! //! Because the model transitions call these very functions, a change to any -//! guard is re-verified against the full bounded state space on the next -//! `cargo test --test model` — the guards cannot drift from the proof. The -//! mutation tests in `tests/model.rs` additionally prove each guard is -//! load-bearing: substituting a broken variant makes the checker produce a -//! counterexample. +//! of them is re-verified against the full bounded state space on the next +//! model run — the decisions cannot drift from the proof. A model expresses +//! a mutation by forging a kernel's INPUT (the listing is "the truth" on an +//! evicting bucket, the fold looks empty, versions are invisible), never by +//! re-implementing the decision, and the mutation tests prove each one +//! load-bearing: the checker must produce a counterexample. //! //! Kernels operate on plain `u64` ranks (a [`WatchCursor`](crate::WatchCursor)'s //! revision, with revisionless cursors ranked 0 by the callers) so they stay @@ -135,7 +138,136 @@ pub fn resume_window_ok(revision: u64, first_sequence: u64) -> bool { /// `tests/model.rs` (`RestoreRead`) and `tests/model_live_watch.rs` /// (`GuardRepair`). pub fn restore_allowed(artifact_revision: u64, local_revision: u64, first_sequence: u64) -> bool { - artifact_revision > local_revision && resume_window_ok(artifact_revision, first_sequence) + restore_ahead(artifact_revision, local_revision) + && resume_window_ok(artifact_revision, first_sequence) +} + +/// The "ahead" half of [`restore_allowed`], on its own: what the main loop +/// re-checks against its own applied cursor (a mid-watch expiry delivered +/// past the resume cursor the watch task checked against). +pub fn restore_ahead(artifact_revision: u64, local_revision: u64) -> bool { + artifact_revision > local_revision +} + +/// Is the bucket's key listing the truth — does "not listed" mean +/// "deleted"? Yes when retention never evicts current values, and also when +/// it can but has evicted nothing yet (first retained revision ≤ 1). +/// Erring toward `false` is safe: it only routes a repair to an artifact. +pub fn listing_is_truth(evicts_current_values: bool, first_revision: u64) -> bool { + !evicts_current_values || resume_window_ok(0, first_revision) +} + +/// The repair a watch armed for an expired cursor, without its payloads +/// (`ExpiryRepair` in `watch_applied`; `None` also when there is no store). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepairMode { + /// Nothing armed: the re-list alone. + None, + /// The key-listing diff. + Relist, + /// The artifact restore. + Restore, + /// Decide by retention. + Auto, +} + +/// What [`plan_repair`] decided for one expiry. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepairPlan { + /// Fall back to the re-list alone. + ReListOnly, + /// The key-listing diff, then the re-list. + Relist, + /// The artifact restore, then a resume from the artifact's cursor. + Restore, + /// Refused: the key-listing diff where the listing isn't the truth + /// would delete keys that merely aged out. The watch fails. + RefuseRelist, +} + +/// THE repair planner. `listing_truth` is [`listing_is_truth`] when the +/// backend reports retention, `None` when it can't say — then `Relist` +/// (chosen explicitly) is trusted as the caller's word, and `Auto` restores. +pub fn plan_repair(mode: RepairMode, listing_truth: Option) -> RepairPlan { + match mode { + RepairMode::None => RepairPlan::ReListOnly, + RepairMode::Relist if listing_truth == Some(false) => RepairPlan::RefuseRelist, + RepairMode::Relist => RepairPlan::Relist, + RepairMode::Restore => RepairPlan::Restore, + RepairMode::Auto if listing_truth == Some(true) => RepairPlan::Relist, + RepairMode::Auto => RepairPlan::Restore, + } +} + +/// A start with no cursor: must the fold be repaired before watching? Yes +/// when it holds data anyway (a torn first checkpoint — a re-list never +/// removes what it doesn't deliver), and, with a restore armed, when the +/// bucket's listing is no longer the truth (a re-list would be incomplete). +pub fn cursorless_start_needs_repair( + mode: RepairMode, + fold_has_data: bool, + listing_truth: Option, +) -> bool { + match mode { + RepairMode::None => false, + _ if fold_has_data => true, + RepairMode::Restore | RepairMode::Auto => listing_truth == Some(false), + RepairMode::Relist => false, + } +} + +/// One side of a key in a restore: absent, or present at a revision +/// (`At(None)`: present but revisionless). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyState { + /// The key isn't there. + Absent, + /// The key is there, at this revision if it has one. + At(Option), +} + +/// What a restore does with one in-scope key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyRestore { + /// Leave the local entry as it is. + Keep, + /// Take the artifact's entry. + TakeArtifact, + /// Delete the local entry. + Delete, +} + +/// THE per-key restore decision. An artifact at cursor `C` is complete +/// together with the log after `C`: a key whose latest write is after `C` +/// may be missing or older in it, and the resume delivers that write. So: +/// +/// - the artifact has the key: take it, unless the local entry is NEWER +/// (revisions decide; without both, only `identical` entries are kept) — +/// never a move backward; +/// - the artifact lacks it, local has it: delete, unless the bucket lists it +/// live (`listed_live`) — then its current value is a write after `C` — +/// never a phantom delete; +/// - neither has it: nothing to do. +pub fn restore_key( + local: KeyState, + artifact: KeyState, + identical: bool, + listed_live: bool, +) -> KeyRestore { + match (local, artifact) { + (KeyState::At(Some(l)), KeyState::At(Some(a))) if l != a => { + if l > a { + KeyRestore::Keep + } else { + KeyRestore::TakeArtifact + } + } + (KeyState::At(_), KeyState::At(_)) if identical => KeyRestore::Keep, + (_, KeyState::At(_)) => KeyRestore::TakeArtifact, + (KeyState::At(_), KeyState::Absent) if listed_live => KeyRestore::Keep, + (KeyState::At(_), KeyState::Absent) => KeyRestore::Delete, + (KeyState::Absent, KeyState::Absent) => KeyRestore::Keep, + } } /// Sequence the watch should start at after applying `revision`. @@ -186,6 +318,74 @@ mod tests { assert_eq!(resume_start_sequence(u64::MAX), None); } + #[test] + fn repair_planner_is_total() { + use RepairMode::*; + for truth in [Option::None, Some(true), Some(false)] { + assert_eq!(plan_repair(None, truth), RepairPlan::ReListOnly); + assert_eq!(plan_repair(Restore, truth), RepairPlan::Restore); + } + assert_eq!(plan_repair(Relist, Some(false)), RepairPlan::RefuseRelist); + assert_eq!(plan_repair(Relist, Some(true)), RepairPlan::Relist); + assert_eq!( + plan_repair(Relist, Option::None), + RepairPlan::Relist, + "caller vouches" + ); + assert_eq!(plan_repair(Auto, Some(true)), RepairPlan::Relist); + assert_eq!(plan_repair(Auto, Some(false)), RepairPlan::Restore); + assert_eq!( + plan_repair(Auto, Option::None), + RepairPlan::Restore, + "unknown: restore" + ); + assert!(listing_is_truth(false, 99)); + assert!( + listing_is_truth(true, 1), + "evicting, but nothing evicted yet" + ); + assert!(!listing_is_truth(true, 2)); + assert!( + cursorless_start_needs_repair(Relist, true, Some(true)), + "data, no cursor" + ); + assert!( + !cursorless_start_needs_repair(None, true, Some(false)), + "nothing armed" + ); + assert!(cursorless_start_needs_repair(Auto, false, Some(false))); + assert!(!cursorless_start_needs_repair(Relist, false, Some(false))); + } + + #[test] + fn restore_key_never_regresses_never_drops_live() { + use KeyRestore::*; + use KeyState::*; + assert_eq!( + restore_key(At(Some(5)), At(Some(3)), false, false), + Keep, + "local newer" + ); + assert_eq!( + restore_key(At(Some(3)), At(Some(5)), false, false), + TakeArtifact + ); + assert_eq!( + restore_key(At(Some(3)), At(Some(3)), true, false), + Keep, + "identical" + ); + assert_eq!(restore_key(At(None), At(None), false, false), TakeArtifact); + assert_eq!(restore_key(Absent, At(Some(1)), false, false), TakeArtifact); + assert_eq!( + restore_key(At(Some(1)), Absent, false, true), + Keep, + "listed live" + ); + assert_eq!(restore_key(At(Some(1)), Absent, false, false), Delete); + assert_eq!(restore_key(Absent, Absent, false, true), Keep); + } + #[test] fn restore_guard_boundaries() { // Local cursor 3 expired (first retained 6): an artifact at 5 resumes diff --git a/src/repair.rs b/src/repair.rs index 4e58b6a..5d87c08 100644 --- a/src/repair.rs +++ b/src/repair.rs @@ -25,13 +25,20 @@ //! a fold (here or in an artifact) holds the former. use std::any::Any; +use std::collections::HashSet; use std::sync::Arc; use async_trait::async_trait; +use tokio::sync::{mpsc, oneshot}; +use tracing::{error, info, warn}; +use crate::applied::{Fold, WatchScope}; use crate::artifact::ExportManifest; -use crate::kv::{KvEntry, KvReader, KvUpdate, VersionToken, WatchCursor}; -use crate::protocol::{restore_allowed, resume_window_ok}; +use crate::kv::{KvEntry, KvError, KvReader, KvUpdate, KvWatcher, VersionToken, WatchCursor}; +use crate::protocol::{ + KeyRestore, KeyState, RepairMode, RepairPlan, cursorless_start_needs_repair, listing_is_truth, + plan_repair, restore_ahead, restore_allowed, restore_key, resume_window_ok, +}; use crate::snapshot::{SnapshotError, SnapshotStore}; /// How [`watch_applied`](crate::watch_applied) repairs the fold when its @@ -88,6 +95,18 @@ pub enum ExpiryRepair { }, } +impl ExpiryRepair { + /// The repair without its payloads, as the protocol kernels take it. + pub(crate) fn mode(&self) -> RepairMode { + match self { + ExpiryRepair::None => RepairMode::None, + ExpiryRepair::Relist(_) => RepairMode::Relist, + ExpiryRepair::Restore { .. } => RepairMode::Restore, + ExpiryRepair::Auto { .. } => RepairMode::Auto, + } + } +} + impl From>> for ExpiryRepair { fn from(reader: Option>) -> Self { match reader { @@ -153,11 +172,6 @@ impl RestoredFold { } } -/// Rank for the restore guard: the cursor's revision, revisionless cursors 0. -pub(crate) fn cursor_rank(c: &WatchCursor) -> u64 { - c.as_u64().unwrap_or(0) -} - /// Does an artifact exported under the `exporter` scope cover every key in /// the `watcher` scope? Both are key-prefix lists (`[""]` is every key). A /// watcher prefix is covered when some exporter prefix is a prefix of it. @@ -196,8 +210,8 @@ pub(crate) fn check_restore( } Some(_) => {} } - let (a, l) = (cursor_rank(&artifact.cursor), cursor_rank(local)); - if a <= l { + let (a, l) = (artifact.cursor.rank(), local.rank()); + if !restore_ahead(a, l) { return Err(format!( "the latest artifact (cursor {a}) is not ahead of the local fold (cursor {l}): \ nothing newer to restore from; the export pipeline is behind this node" @@ -231,57 +245,48 @@ impl RestoreOp { } } -/// Does the local entry already hold the artifact's value or a newer one? -/// Revisions decide when both have one; otherwise only identical entries -/// count. -fn local_is_current(local: &KvEntry, artifact: &KvEntry) -> bool { - match (local.version.as_u64(), artifact.version.as_u64()) { - (Some(l), Some(a)) if l != a => l > a, - _ => local.version == artifact.version && local.value == artifact.value, - } -} - /// The in-scope ops that bring the `local` fold to the `artifact` fold's /// state at its cursor `C`, without ever moving a key backward or deleting a /// live one. Keys only, in key order — values are read back in bounded /// chunks ([`materialize`]), so a restore never holds the whole changed set's /// values at once (the same discipline as the key-listing diff). /// -/// An artifact is complete together with the log after its cursor: every -/// RETAINED message at or below `C` is in it, but a key whose latest write is -/// after `C` may be missing or older (its exporter was mid catch-up). The -/// resume from `C` delivers those. So, per in-scope key: -/// -/// - **artifact has it**: `Put`, unless the local entry is already the same -/// or NEWER — then the key's latest write must be after `C` (the artifact -/// would hold it otherwise), and the resume delivers it; taking the older -/// artifact value would move the key backward. -/// - **artifact lacks it, local has it**: `Delete`, unless the bucket lists -/// it as `live` — then its current value is a write after `C` (again, the -/// artifact would hold it otherwise), which the resume delivers; deleting -/// it would drop a live key. A key not listed was really deleted (or its -/// delete marker evicted), which is exactly what the exporter folded. -/// -/// Out-of-scope keys are untouched on both sides. +/// Each in-scope key is decided by [`restore_key`], which the repair model +/// (`tests/model_repair.rs`) runs too. Out-of-scope keys are untouched on +/// both sides. pub(crate) fn restore_diff( local: &S, artifact: &S, prefixes: &[String], - live: &std::collections::HashSet, + live: &HashSet, ) -> Result, SnapshotError> { let mut ops = Vec::new(); for prefix in prefixes { + // Every key the artifact holds. artifact.for_each_in_range(prefix, |entry| { - let current = local - .get(&entry.key)? - .is_some_and(|l| local_is_current(&l, &entry)); - if !current { + let at = KeyState::At(entry.version.as_u64()); + let (state, identical) = match local.get(&entry.key)? { + Some(l) => ( + KeyState::At(l.version.as_u64()), + l.version == entry.version && l.value == entry.value, + ), + None => (KeyState::Absent, false), + }; + let listed = live.contains(&entry.key); + if restore_key(state, at, identical, listed) == KeyRestore::TakeArtifact { ops.push(RestoreOp::Put(entry.key)); } Ok(()) })?; + // Local keys the artifact lacks. The kernel is asked first, as if the + // artifact lacked the key, so the artifact is read only for keys that + // would be deleted. local.for_each_in_range(prefix, |entry| { - if !live.contains(&entry.key) && artifact.get(&entry.key)?.is_none() { + let state = KeyState::At(entry.version.as_u64()); + let listed = live.contains(&entry.key); + if restore_key(state, KeyState::Absent, false, listed) == KeyRestore::Delete + && artifact.get(&entry.key)?.is_none() + { ops.push(RestoreOp::Delete(entry.key)); } Ok(()) @@ -317,6 +322,628 @@ pub(crate) fn materialize( .collect() } +// --------------------------------------------------------------------------- +// The repair flow. Two tasks share it: the watch task, which owns the watch +// and decides on a repair when the cursor expires, and `watch_applied`'s main +// loop, which owns the fold and applies the repair to it. The watch task +// hands the repair over (`RepairRequest`) and stays parked on the ack/reply +// until the fold has it, so the repair is strictly ordered between everything +// delivered before the expiry and everything the next watch delivers. +// --------------------------------------------------------------------------- + +/// A cursor-expired repair handoff from the watch task to the main loop. +pub(crate) enum RepairRequest { + /// The key-listing diff: the bucket's live keys for the watch scope. The + /// main loop applies synthetic deletes for in-scope keys missing from it, + /// then acks so the watch task can start the fallback re-list. + Relist { + live_keys: Vec, + ack: oneshot::Sender<()>, + }, + /// The artifact restore: a verified artifact fold, and the bucket's live + /// keys for the scope (listed after the fetch). The main loop folds the + /// in-scope difference, advances to the artifact's cursor, and replies + /// with it so the watch task resumes from there. + Restore { + restored: RestoredFold, + live: HashSet, + reply: oneshot::Sender>, + }, +} + +/// What the watch task needs to repair an expiry: the caller's chosen +/// repair, and the channel into the main loop that owns the fold. +pub(crate) struct RepairHandle { + mode: ExpiryRepair, + tx: mpsc::Sender>, +} + +/// The repair the watch task settled on for one expiry. +enum Plan<'a, S> { + /// Fall back to the re-list alone (nothing armed). + ReListOnly, + /// The key-listing diff, then the re-list. + Relist(&'a Arc, &'a mpsc::Sender>), + /// The artifact restore, then a resume from the artifact's cursor. + Restore( + &'a Arc, + &'a Arc>, + &'a mpsc::Sender>, + ), +} + +/// Arm `mode` for a watch: the watch task's handle and the main loop's +/// receiving end, both `None` when there is nothing to run (no repair, or no +/// store to run it against — both repairs edit the fold). +/// +/// A restore replaces the fold, so asking for one without a fold is a +/// contradiction: on a bounded log such a consumer would silently hold +/// NATS's retained view (missing everything evicted) while believing it +/// repairs. That is refused up front; `ExpiryRepair::None` is the explicit +/// way to accept the retained view. +#[allow(clippy::type_complexity)] +pub(crate) fn arm( + mode: ExpiryRepair, + has_store: bool, +) -> Result< + ( + Option>, + Option>>, + ), + KvError, +> { + match mode.mode() { + RepairMode::None => Ok((None, None)), + _ if has_store => { + let (tx, rx) = mpsc::channel(1); + Ok((Some(RepairHandle { mode, tx }), Some(rx))) + } + RepairMode::Restore | RepairMode::Auto => Err(KvError::WatchError( + "ExpiryRepair::Restore / ::Auto repair the fold, so they need a store: a consumer \ + without one cannot repair a bounded log. Pass a SnapshotStore, or \ + ExpiryRepair::None to accept NATS's retained view" + .into(), + )), + RepairMode::Relist => Ok((None, None)), + } +} + +/// [`listing_is_truth`] for the watcher's live retention; `None` when the +/// backend can't say. +async fn listing_truth(watcher: &dyn KvWatcher) -> Result, KvError> { + Ok(watcher + .retention() + .await? + .map(|r| listing_is_truth(r.evicts_current_values, r.first_revision))) +} + +/// The watch task: run the underlying watch for `scope`, resuming from +/// `resume` when it carries a position, with cursor-expiry repair: an expiry +/// (at resume, or mid-watch from the floor guard) runs the planned repair and +/// then either falls back to the full-scope re-list (Relist / nothing armed) +/// or resumes from the restored artifact's cursor (Restore) — which can +/// itself expire later and go round again, each time only for a strictly +/// newer artifact. +/// +/// `unanchored`: the fold holds data but no cursor (a torn first checkpoint, +/// or a populated store started without one). A re-list alone never removes +/// what it doesn't deliver, so that start is repaired like an expiry at +/// revision 0 before watching. +pub(crate) async fn run_watch( + watcher: Arc, + scope: WatchScope, + resume: Option, + unanchored: bool, + repair: Option>, + tx: mpsc::Sender, +) -> Result<(), KvError> { + let (watcher, scope) = (watcher.as_ref(), &scope); + // Resume only when the cursor carries a real position; an absent or `none()` + // cursor falls through to a full watch. Binding `cursor` here makes "we have a + // resume position" structural — there is no separate bool whose truth a later + // edit could let drift from the `Some`. + let mut resume = resume.filter(|c| !c.is_none()); + + // A cursor-less start that must be repaired before it watches + // ([`cursorless_start_needs_repair`]). Retention is read only when the + // decision, or the warning, depends on it. + let mode = repair.as_ref().map_or(RepairMode::None, |h| h.mode.mode()); + let mut repair_first = false; + if resume.is_none() { + let truth = match mode { + RepairMode::Relist => None, + RepairMode::Restore | RepairMode::Auto if unanchored => None, + _ => listing_truth(watcher).await?, + }; + repair_first = cursorless_start_needs_repair(mode, unanchored, truth); + if repair_first && unanchored { + warn!( + "the fold holds data but no cursor; repairing it like an expired cursor \ + before watching (a re-list alone never removes what it doesn't deliver)" + ); + } else if repair_first { + warn!( + "no resume cursor, and the bucket has already evicted current values: a \ + re-list would seed an incomplete fold; restoring from the latest artifact" + ); + } else if mode == RepairMode::None && truth == Some(false) { + // Nothing armed: the re-list stands as the fold, and lacks + // whatever aged out. Nothing will repair it. + warn!( + "no resume cursor and no repair armed, on a bucket that has already \ + evicted current values: the re-list lacks whatever aged out. Wire a \ + store and ExpiryRepair::Auto to seed from an artifact" + ); + } + } + + loop { + let cursor = if std::mem::take(&mut repair_first) { + WatchCursor::none() + } else { + let Some(cursor) = resume.take() else { + return watch_scope(watcher, scope, tx).await; + }; + match watch_scope_from(watcher, scope, &cursor, tx.clone()).await { + Err(KvError::CursorExpired) => cursor, + other => return other, + } + }; + match plan(watcher, repair.as_ref()).await? { + Plan::ReListOnly => { + warn!( + "watch cursor expired with no repair armed (needs a store and a reader or \ + restore source); falling back to the re-list alone — keys deleted during \ + the gap may persist in the fold" + ); + return watch_scope(watcher, scope, tx).await; + } + Plan::Relist(reader, repairs) => { + warn!( + "watch cursor expired; resyncing stale keys, then falling back to the full re-list" + ); + resync_stale_keys(scope, reader, repairs).await?; + return watch_scope(watcher, scope, tx).await; + } + Plan::Restore(reader, source, repairs) => { + warn!("watch cursor expired; restoring the fold from the latest artifact"); + let restored = + restore_from_artifact(watcher, scope, reader, source, repairs, &cursor) + .await + .map_err(|e| match e { + KvError::WatchError(msg) if cursor.is_none() => { + KvError::WatchError(format!( + "{msg}. This node has no cursor, so it can only seed from an \ + artifact. If none exists yet (first deploy onto a bucket that \ + already evicted values), start one node with ExpiryRepair::None \ + — accepting that values which already aged out are gone — and \ + publish an export from it" + )) + } + other => other, + })?; + resume = Some(restored); + } + } + } +} + +/// The full-scope (state-sync re-list) watch for `scope`. +async fn watch_scope( + watcher: &dyn KvWatcher, + scope: &WatchScope, + tx: mpsc::Sender, +) -> Result<(), KvError> { + match scope { + WatchScope::All => watcher.watch_all(tx).await, + WatchScope::Prefix(prefix) => watcher.watch_prefix(prefix, tx).await, + WatchScope::Prefixes(prefixes) => { + let refs: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + watcher.watch_prefixes(&refs, tx).await + } + } +} + +/// The delta watch for `scope`, resuming after `cursor`. +async fn watch_scope_from( + watcher: &dyn KvWatcher, + scope: &WatchScope, + cursor: &WatchCursor, + tx: mpsc::Sender, +) -> Result<(), KvError> { + match scope { + WatchScope::All => watcher.watch_all_from(cursor, tx).await, + WatchScope::Prefix(prefix) => watcher.watch_prefix_from(prefix, cursor, tx).await, + WatchScope::Prefixes(prefixes) => { + let refs: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + watcher.watch_prefixes_from(&refs, cursor, tx).await + } + } +} + +/// Decide how to repair one expiry ([`plan_repair`]), reading the bucket's +/// retention live when the choice depends on it. +async fn plan<'a, S>( + watcher: &dyn KvWatcher, + repair: Option<&'a RepairHandle>, +) -> Result, KvError> { + let Some(h) = repair else { + return Ok(Plan::ReListOnly); + }; + let mode = h.mode.mode(); + let truth = match mode { + RepairMode::Relist | RepairMode::Auto => listing_truth(watcher).await?, + RepairMode::None | RepairMode::Restore => None, + }; + Ok(match (plan_repair(mode, truth), &h.mode) { + (RepairPlan::ReListOnly, _) => Plan::ReListOnly, + (RepairPlan::RefuseRelist, _) => { + let msg = "watch cursor expired on a bucket whose retention evicts current \ + values (max_age, per-message TTL, or discard:old): its key listing \ + can't tell a deleted key from an aged-out one, so the key-listing \ + resync would delete valid keys. Refusing. Repair from artifacts with \ + ExpiryRepair::Restore or ExpiryRepair::Auto (or accept a re-list-only \ + fallback with ExpiryRepair::None)"; + error!(msg); + return Err(KvError::WatchError(msg.into())); + } + (RepairPlan::Relist, ExpiryRepair::Relist(reader) | ExpiryRepair::Auto { reader, .. }) => { + Plan::Relist(reader, &h.tx) + } + ( + RepairPlan::Restore, + ExpiryRepair::Restore { reader, restore } | ExpiryRepair::Auto { reader, restore }, + ) => Plan::Restore(reader, restore, &h.tx), + (plan, _) => unreachable!("plan_repair chose {plan:?}, which mode {mode:?} can't run"), + }) +} + +/// The artifact restore, watch-task half: check the newest artifact before +/// downloading it (ahead of `local`, covers the scope, still inside +/// retention), download it, check what actually arrived, then hand it to the +/// main loop and wait for the cursor it restored to. +async fn restore_from_artifact( + watcher: &dyn KvWatcher, + scope: &WatchScope, + reader: &Arc, + source: &Arc>, + repairs: &mpsc::Sender>, + local: &WatchCursor, +) -> Result { + let prefixes = scope.prefixes(); + let fail = |msg: String| { + error!(%msg, "cursor-expiry restore refused; no safe recovery"); + KvError::WatchError(format!("cursor-expired restore: {msg}")) + }; + + let first = watcher.retention().await?.map(|r| r.first_revision); + let latest = source + .latest() + .await + .map_err(|e| fail(format!("reading the latest artifact failed: {e}")))?; + check_restore(&latest, local, first, &prefixes).map_err(fail)?; + + let restored = source + .fetch() + .await + .map_err(|e| fail(format!("fetching the latest artifact failed: {e}")))?; + // The pointer may have advanced between the peek and the download, and + // retention may have moved: check what actually arrived, against now. + let first = watcher.retention().await?.map(|r| r.first_revision); + check_restore(restored.manifest(), local, first, &prefixes).map_err(fail)?; + let age_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|now| { + now.as_secs() + .saturating_sub(restored.manifest().created_at_unix) + }) + .unwrap_or(0); + info!( + cursor = ?restored.manifest().cursor, + age_secs, + first_retained = ?first, + "restoring the fold from the latest artifact" + ); + + // The bucket's live keys, listed after the artifact's cursor is fixed: + // the restore never deletes one of these (see `restore_diff`). A failed + // listing fails the watch, like the key-listing repair's. + let mut live = HashSet::new(); + for prefix in &prefixes { + let keys = reader + .keys(prefix) + .await + .map_err(|e| fail(format!("listing live keys under {prefix:?} failed: {e}")))?; + live.extend(keys); + } + + let (reply_tx, reply_rx) = oneshot::channel(); + repairs + .send(RepairRequest::Restore { + restored, + live, + reply: reply_tx, + }) + .await + .map_err(|_| KvError::WatchError("watch loop ended during a restore".into()))?; + reply_rx + .await + .map_err(|_| KvError::WatchError("watch loop dropped a restore reply".into()))? +} + +/// Cursor-expired stale-key resync (the key-listing diff), run BEFORE the +/// fallback watch is established: list the scope's live keys, hand them to the +/// main loop (which diffs them against the fold and applies synthetic +/// deletes), and wait for the ack. That ordering — deletes applied, then +/// fallback watch armed — is what makes a delete-then-recreate during the gap +/// converge: the synthetic delete always lands before the re-list put. +/// +/// Sound only on a bucket that never evicts current values; `plan` refuses it +/// otherwise. +/// +/// A FAILED listing is **fatal** — it fails the watch rather than degrading. +/// The resync is load-bearing for the "stale, never corrupt" convergence +/// guarantee: a silently degraded resync leaves the fold holding keys the +/// bucket deleted, with one warn line as the only witness (`tests/model.rs` +/// proves this divergence reachable under degrade semantics). Failing the +/// watch turns the violated guarantee into a visible error; the caller's +/// restart re-resumes, hits `CursorExpired` again, and retries the resync from +/// scratch. +async fn resync_stale_keys( + scope: &WatchScope, + reader: &Arc, + repairs: &mpsc::Sender>, +) -> Result<(), KvError> { + let mut live_keys = Vec::new(); + for prefix in scope.prefixes() { + match reader.keys(&prefix).await { + Ok(keys) => live_keys.extend(keys), + Err(e) => { + return Err(KvError::WatchError(format!( + "cursor-expired resync failed listing live keys under {prefix:?}: {e}; \ + failing the watch rather than silently keeping stale keys" + ))); + } + } + } + let (ack_tx, ack_rx) = oneshot::channel(); + if repairs + .send(RepairRequest::Relist { + live_keys, + ack: ack_tx, + }) + .await + .is_ok() + { + // A dropped ack (main loop shutting down) just means the fallback watch + // is about to die with it; nothing to recover. + let _ = ack_rx.await; + } + Ok(()) +} + +/// Fail the watch from inside a repair: the fold is either unchanged or +/// part-way through a repair that commits under its old (expired) cursor, so +/// the caller's restart re-runs the whole expiry path. +fn repair_fatal(msg: String) -> KvError { + error!(%msg, "cursor-expiry repair failed; aborting watch"); + KvError::WatchError(msg) +} + +/// The main-loop half: apply one repair to the fold. `backlog` is the watch +/// task's update channel and `prefixes` the watch scope. An `Err` is fatal to +/// the watch. +pub(crate) async fn fold_in( + fold: &mut Fold, + req: RepairRequest, + backlog: &mut mpsc::Receiver, + prefixes: &[String], +) -> Result<(), KvError> +where + U: Send, + S: SnapshotStore + Send + 'static, + P: FnMut(&KvUpdate) -> Option + Send, + A: FnMut(Vec) + Send, + O: FnMut(WatchCursor) + Send, +{ + // Fold everything the watch task delivered before it asked for this + // repair. A floor-guard trip ends a live watch mid-stream with updates + // still buffered in the channel: the repair must see them (it diffs the + // fold) and supersede them (one applied after the repair would resurrect + // pre-repair state, and could even move the cursor backward). The watch + // task is parked on the ack/reply, so this drains exactly that backlog. + while let Ok(u) = backlog.try_recv() { + if fold.ingest(u) { + fold.flush().await?; + } + } + // Both repairs diff the STORE, so it must hold everything delivered. A + // transient store failure re-queues the batch, which the diff can't see: + // it would miss a re-queued put that a deletion during the gap has since + // removed, and the put would then commit and resurrect the key. + fold.settle().await?; + match req { + RepairRequest::Relist { live_keys, ack } => { + relist(fold, &live_keys, prefixes).await?; + // Ack AFTER the deletes are applied: the watch task is holding + // the fallback watch until it hears back, which is what orders + // deletes before the re-list (tests/model_resync_order.rs proves + // the barrier load-bearing). If the flush's STORE apply failed + // transiently, the deletes sit re-queued at the FRONT of the raw + // batch — still strictly before any re-list put in the eventual + // cumulative commit, and the domain apply saw them before this ack + // either way. + let _ = ack.send(()); + Ok(()) + } + RepairRequest::Restore { + restored, + live, + reply, + } => { + let target = restored.manifest().cursor.clone(); + // The authoritative ahead check (the watch task's ran against its + // resume cursor; a mid-watch expiry delivered past that). A + // restore never moves the fold backward. + if !restore_ahead(target.rank(), fold.applied().rank()) { + let msg = format!( + "cursor-expired restore: the latest artifact (cursor {target:?}) is not \ + ahead of the fold (cursor {:?}); nothing newer to restore from", + fold.applied() + ); + error!(%msg, "cursor-expiry repair refused"); + // The watch task fails with this; its channel closing ends the + // loop. + let _ = reply.send(Err(KvError::WatchError(msg))); + return Ok(()); + } + restore(fold, restored, live, prefixes).await?; + let _ = reply.send(Ok(target)); + Ok(()) + } + } +} + +/// The key-listing diff, main-loop half: synthetic deletes for in-scope keys +/// the fold holds and the bucket no longer lists — they vanished during the +/// gap (their delete markers evicted with the cursor), and the re-list can't +/// deliver a delete. +async fn relist( + fold: &mut Fold, + live_keys: &[String], + prefixes: &[String], +) -> Result<(), KvError> +where + U: Send, + S: SnapshotStore + Send + 'static, + P: FnMut(&KvUpdate) -> Option + Send, + A: FnMut(Vec) + Send, + O: FnMut(WatchCursor) + Send, +{ + let live: HashSet<&str> = live_keys.iter().map(String::as_str).collect(); + let mut stale: Vec = Vec::new(); + if let Some(st) = fold.store() { + for prefix in prefixes { + // Stream the fold's keys rather than `range()`, which buffers every + // in-scope entry — values included — into one Vec. On an on-disk + // backend holding a fold larger than RAM (the case those backends + // exist for), an All-scope resync would materialize the entire + // fold on the repair path. Only the keys matter. + if let Err(e) = st.for_each_in_range(prefix, |entry| { + if !live.contains(entry.key.as_str()) { + stale.push(entry.key); + } + Ok(()) + }) { + // FATAL, not a degrade: an incomplete diff silently leaves + // deleted keys in the fold forever (tests/model.rs proves the + // divergence reachable under degrade semantics). Fail the + // watch; the restart re-runs the resume → expiry → resync from + // scratch. + warn!(error = %e, prefix = %prefix, + "resync fold scan failed; aborting watch rather than diverging"); + return Err(KvError::WatchError(format!( + "cursor-expired resync failed listing fold prefix {prefix:?}: {e}" + ))); + } + } + } + // Overlapping prefixes can list a key twice. + stale.sort_unstable(); + stale.dedup(); + if !stale.is_empty() { + warn!( + stale = stale.len(), + "cursor-expired resync: deleting keys that vanished during the gap" + ); + } + for key in stale { + // Synthetic: carries no revision (unknown version) and so never + // advances the cursor. + fold.correct(KvUpdate::Delete { + key, + version: VersionToken::unknown(), + }); + } + fold.flush().await +} + +/// The artifact restore, main-loop half: replace the in-scope fold with the +/// artifact's ([`restore_diff`]), then take its cursor. +async fn restore( + fold: &mut Fold, + mut restored: RestoredFold, + live: HashSet, + prefixes: &[String], +) -> Result<(), KvError> +where + U: Send, + S: SnapshotStore + Send + 'static, + P: FnMut(&KvUpdate) -> Option + Send, + A: FnMut(Vec) + Send, + O: FnMut(WatchCursor) + Send, +{ + let target = restored.manifest().cursor.clone(); + let Some(st) = fold.take_store() else { + unreachable!("repairs are armed only with a store") + }; + let (prefixes, max) = (prefixes.to_vec(), fold.batch_cap()); + let (st, r, diff) = tokio::task::spawn_blocking(move || { + let diff = restore_diff(&st, restored.fold(), &prefixes, &live); + (st, restored, diff) + }) + .await + .map_err(|e| repair_fatal(format!("cursor-expired restore: diff task panicked: {e}")))?; + fold.put_store(st); + restored = r; + let diff = diff.map_err(|e| { + repair_fatal(format!( + "cursor-expired restore: diffing the fold against the artifact failed: {e}" + )) + })?; + info!( + from = ?fold.applied(), + to = ?target, + changed = diff.len(), + "cursor-expired restore: replacing the in-scope fold with the artifact's" + ); + // Fold the difference through parse/apply/store in `max`-sized chunks, + // values read back per chunk. These carry no stream position: every chunk + // commits under the old cursor, so a crash part-way re-runs the + // (idempotent) restore on restart. + let mut ops = diff.into_iter(); + loop { + let chunk: Vec<_> = ops.by_ref().take(max).collect(); + if chunk.is_empty() { + break; + } + let (r, updates) = tokio::task::spawn_blocking(move || { + let updates = materialize(restored.fold(), chunk); + (restored, updates) + }) + .await + .map_err(|e| repair_fatal(format!("cursor-expired restore: read task panicked: {e}")))?; + restored = r; + let updates = updates.map_err(|e| { + repair_fatal(format!( + "cursor-expired restore: reading the artifact failed: {e}" + )) + })?; + for u in updates { + fold.correct(u); + } + fold.flush().await?; + } + // Now the fold IS the artifact's (in scope): take its cursor, committed to + // the store with the final chunk's re-queued remainder, if any. + fold.advance_to(target); + fold.flush().await?; + // Closing an LSM can block; drop the temporary fold and its scratch dir + // off the async thread. + let _ = tokio::task::spawn_blocking(move || drop(restored)).await; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/snapshot.rs b/src/snapshot.rs index 0c9eec8..c4fb9cd 100644 --- a/src/snapshot.rs +++ b/src/snapshot.rs @@ -45,7 +45,8 @@ use std::io::{self, Write}; use std::path::{Path, PathBuf}; use crate::artifact::{ExportManifest, ExportStage, verify_and_stage_import}; -use crate::kv::{KvEntry, KvUpdate, VersionToken, WatchCursor}; +use crate::kv::{KvEntry, KvUpdate, Retention, VersionToken, WatchCursor}; +use crate::protocol::{RepairMode, RepairPlan, listing_is_truth, plan_repair}; const MAGIC: &[u8; 4] = b"PGSS"; // v2: Put/Delete records store the version as length-prefixed raw bytes instead @@ -104,27 +105,39 @@ pub struct Snapshot { } impl Snapshot { - /// Keys present in this snapshot but absent from a fresh scan result. + /// Keys present in this snapshot but absent from a fresh scan result: + /// after a cursor-expired fallback to a full `watch_all()`, the keys to + /// delete, because the re-list can't deliver deletes whose markers were + /// evicted with the cursor. /// - /// After a cursor-expired fallback to full `watch_all()`, callers should - /// compare the snapshot against the live key set and emit synthetic - /// `Delete` events for stale keys to ensure convergence. - /// - /// Only on a bucket that never evicts current values (`discard: new`, no - /// `max_age`). Where retention evicts current values, a key absent from - /// the scan may simply have aged out, and deleting it loses data; repair - /// from an exported artifact instead - /// ([`ExpiryRepair`](crate::ExpiryRepair)). - pub fn stale_keys<'a, I>(&'a self, current_keys: I) -> Vec<&'a str> + /// `retention` is the bucket's ([`KvWatcher::retention`](crate::KvWatcher::retention)). + /// Returns `None` when it says the scan isn't the truth — retention evicts + /// current values and has evicted some — because a key absent from the + /// scan may then have merely aged out, and deleting it loses data; repair + /// from an exported artifact instead ([`ExpiryRepair`](crate::ExpiryRepair)). + /// With `retention` unknown (`None`), the caller vouches that it doesn't + /// evict current values. The same rule, from the same planner, as + /// `ExpiryRepair::Relist`. + pub fn stale_keys<'a, I>( + &'a self, + current_keys: I, + retention: Option, + ) -> Option> where I: IntoIterator, { + let truth = retention.map(|r| listing_is_truth(r.evicts_current_values, r.first_revision)); + if plan_repair(RepairMode::Relist, truth) == RepairPlan::RefuseRelist { + return None; + } let current: HashSet<&str> = current_keys.into_iter().collect(); - self.entries - .keys() - .filter(|k| !current.contains(k.as_str())) - .map(|k| k.as_str()) - .collect() + Some( + self.entries + .keys() + .filter(|k| !current.contains(k.as_str())) + .map(|k| k.as_str()) + .collect(), + ) } } @@ -409,6 +422,13 @@ pub trait SnapshotStore: Sized + Send { Ok(()) } + /// Does the fold hold any entry? The default reads the whole fold through + /// [`range`](Self::range); a backend whose fold can be large should + /// override it to read at most one entry. + fn has_entries(&self) -> Result { + Ok(!self.range("")?.is_empty()) + } + /// The most recently applied (and durably persisted) resume cursor — /// [`WatchCursor::none`] when nothing has been applied. fn cursor(&self) -> WatchCursor; @@ -606,6 +626,10 @@ impl SnapshotStore for AppendLogSnapshot { Ok(out) } + fn has_entries(&self) -> Result { + Ok(!self.entries.is_empty()) + } + fn cursor(&self) -> WatchCursor { self.cursor.clone() } @@ -1611,20 +1635,54 @@ mod tests { let snap = load(&path).unwrap().unwrap(); // Simulate a fresh scan that only has "node.a" and "node.c" - let mut stale = snap.stale_keys(["node.a", "node.c"]); + let mut stale = snap.stale_keys(["node.a", "node.c"], None).unwrap(); stale.sort(); assert_eq!(stale, vec!["node.b"]); // All keys present → no stale - let stale = snap.stale_keys(["node.a", "node.b", "node.c"]); + let stale = snap + .stale_keys(["node.a", "node.b", "node.c"], None) + .unwrap(); assert!(stale.is_empty()); // No keys present → all stale - let mut stale: Vec<&str> = snap.stale_keys(std::iter::empty::<&str>()); + let mut stale: Vec<&str> = snap.stale_keys(std::iter::empty::<&str>(), None).unwrap(); stale.sort(); assert_eq!(stale, vec!["node.a", "node.b", "node.c"]); } + /// Where retention evicts current values and has evicted some, the scan + /// isn't the truth: no stale set, rather than one that deletes keys that + /// merely aged out. + #[test] + fn stale_keys_refuses_a_scan_that_isnt_the_truth() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.snap"); + let mut w = SnapshotWriter::open(&path, u64::MAX).unwrap(); + w.write_update(&put("node.a", b"v1", 1)).unwrap(); + w.checkpoint(&cursor(1)).unwrap(); + drop(w); + let snap = load(&path).unwrap().unwrap(); + let retention = |evicts, first| { + Some(Retention { + evicts_current_values: evicts, + first_revision: first, + }) + }; + + assert_eq!(snap.stale_keys([], retention(true, 5)), None, "evicted"); + assert_eq!( + snap.stale_keys([], retention(true, 1)), + Some(vec!["node.a"]), + "evicting retention that has evicted nothing yet" + ); + assert_eq!( + snap.stale_keys([], retention(false, 5)), + Some(vec!["node.a"]), + "keeps current values" + ); + } + #[test] fn non_u64_version_token_round_trips() { // Regression: Put/Delete records store the version as length-prefixed raw diff --git a/src/snapshot_fjall.rs b/src/snapshot_fjall.rs index 97b6a17..4ff0ae2 100644 --- a/src/snapshot_fjall.rs +++ b/src/snapshot_fjall.rs @@ -132,7 +132,7 @@ impl Default for FjallConfig { } } -/// On-disk durable fold backed by fjall. See the [module docs](self). +/// On-disk durable fold backed by fjall. See the module docs in `src/snapshot_fjall.rs`. pub struct FjallSnapshot { // fjall 3 renamed its types: the database root is `Database` (was `Keyspace`) // and each named partition is a `Keyspace` (was `PartitionHandle`). @@ -516,6 +516,13 @@ impl SnapshotStore for FjallSnapshot { Ok(()) } + fn has_entries(&self) -> Result { + match self.data.prefix(b"").next() { + Some(guard) => guard.into_inner().map(|_| true).map_err(map_fjall), + None => Ok(false), + } + } + fn cursor(&self) -> WatchCursor { self.cursor.clone() } diff --git a/src/snapshot_rocksdb.rs b/src/snapshot_rocksdb.rs index 466912d..00786be 100644 --- a/src/snapshot_rocksdb.rs +++ b/src/snapshot_rocksdb.rs @@ -200,7 +200,7 @@ impl Default for RocksDbConfig { } } -/// On-disk durable fold backed by RocksDB. See the [module docs](self). +/// On-disk durable fold backed by RocksDB. See the module docs in `src/snapshot_rocksdb.rs`. pub struct RocksDbSnapshot { // Arc so `reader()` handles share the instance: RocksDB serves reads from // `&DB` concurrently with writes, and `DB` is `Send + Sync`. @@ -599,6 +599,16 @@ impl SnapshotStore for RocksDbSnapshot { scan_prefix(&self.db, prefix, f) } + fn has_entries(&self) -> Result { + let mut it = self + .db + .iterator_cf(cf(&self.db, DATA_CF)?, IteratorMode::Start); + match it.next() { + Some(item) => item.map(|_| true).map_err(map_rocksdb), + None => Ok(false), + } + } + fn cursor(&self) -> WatchCursor { self.cursor.clone() } diff --git a/src/transport.rs b/src/transport.rs index 96ef56d..232e6f7 100644 --- a/src/transport.rs +++ b/src/transport.rs @@ -114,12 +114,6 @@ pub enum PublishOutcome { }, } -/// Total order for the monotonic pointer guard. Revisionless cursors rank 0: -/// a real cursor always supersedes an empty one, never the reverse. -fn cursor_rank(c: &WatchCursor) -> u64 { - c.as_u64().unwrap_or(0) -} - /// Bound one object-store await by `limit`. async fn timed_by( what: &str, @@ -264,7 +258,7 @@ impl ObjectStoreTransport { ObjPath::from(format!( "{}/{key}.payloads/{:016x}-{hex}.tar", self.prefix, - cursor_rank(cursor) + cursor.rank() )) } @@ -335,9 +329,9 @@ impl ObjectStoreTransport { // rank-less and replaced. let existing = manifest_from_slice(&bytes).ok(); let observed = PointerState::Present { - rank: existing.as_ref().map(|m| cursor_rank(&m.cursor)), + rank: existing.as_ref().map(|m| m.cursor.rank()), }; - if !pointer_publish_allowed(&observed, cursor_rank(new_cursor)) { + if !pointer_publish_allowed(&observed, new_cursor.rank()) { let existing = existing.expect("refusal implies a parseable, newer pointer"); return Ok(PublishOutcome::SupersededByNewer { @@ -612,7 +606,7 @@ impl ArtifactTransport for ObjectStoreTransport { return Ok(0); }; let keep = self.payload_path(key, ¤t.cursor, &pointer); - let pointer_rank = cursor_rank(¤t.cursor); + let pointer_rank = current.cursor.rank(); let cutoff_millis = std::time::SystemTime::now() .checked_sub(grace) .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) diff --git a/tests/common/crash.rs b/tests/common/crash.rs new file mode 100644 index 0000000..0c8c488 --- /dev/null +++ b/tests/common/crash.rs @@ -0,0 +1,86 @@ +//! Crash injection for the transport tiers. + +/// An [`object_store::ObjectStore`] that, once armed, fails the next PUT of a +/// `*.manifest.json` object — the process crash in the window between +/// `upload`'s payload multipart (already complete) and its sibling-manifest +/// PUT. Everything else delegates to the wrapped store. +#[derive(Debug)] +pub struct ManifestPutCrash { + pub inner: std::sync::Arc, + pub armed: std::sync::atomic::AtomicBool, +} + +impl std::fmt::Display for ManifestPutCrash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ManifestPutCrash({})", self.inner) + } +} + +#[async_trait::async_trait] +impl object_store::ObjectStore for ManifestPutCrash { + async fn put_opts( + &self, + location: &object_store::path::Path, + payload: object_store::PutPayload, + opts: object_store::PutOptions, + ) -> object_store::Result { + if location.as_ref().ends_with(".manifest.json") + && self.armed.swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Err(object_store::Error::Generic { + store: "ManifestPutCrash", + source: "injected crash between payload and manifest PUT".into(), + }); + } + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &object_store::path::Path, + opts: object_store::PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts( + &self, + location: &object_store::path::Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: futures::stream::BoxStream< + 'static, + object_store::Result, + >, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + options: object_store::CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} diff --git a/tests/common/minio.rs b/tests/common/minio.rs new file mode 100644 index 0000000..4e01b98 --- /dev/null +++ b/tests/common/minio.rs @@ -0,0 +1,128 @@ +//! A throwaway `minio` with a pre-created bucket, one per test, killed on +//! drop. `MINIO_BIN` / `MC_BIN` point at explicit paths outside an activated +//! mise shell. + +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use tempfile::TempDir; + +use super::nats::free_port; + +pub const MINIO_USER: &str = "minioadmin"; +pub const MINIO_PASSWORD: &str = "minioadmin"; +pub const MINIO_BUCKET: &str = "test-bucket"; + +/// A running `minio` with [`MINIO_BUCKET`] pre-created via `mc`. Killed on +/// drop. The `mc mb` retry loop doubles as the readiness probe. +pub struct TestMinio { + child: Child, + pub endpoint: String, + _data_dir: TempDir, +} + +impl TestMinio { + pub async fn start() -> TestMinio { + let minio_bin = std::env::var("MINIO_BIN").unwrap_or_else(|_| "minio".to_string()); + let mc_bin = std::env::var("MC_BIN").unwrap_or_else(|_| "mc".to_string()); + let api_port = free_port(); + let console_port = free_port(); + let data_dir = tempfile::tempdir().expect("create minio data dir"); + + let child = Command::new(&minio_bin) + .args([ + "server", + data_dir.path().to_str().expect("utf-8 data path"), + "--address", + &format!("127.0.0.1:{api_port}"), + "--console-address", + &format!("127.0.0.1:{console_port}"), + ]) + .env("MINIO_ROOT_USER", MINIO_USER) + .env("MINIO_ROOT_PASSWORD", MINIO_PASSWORD) + .env("MINIO_BROWSER", "off") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `{minio_bin}`: {e}. Is minio installed? \ + Run `mise install` or set MINIO_BIN." + ) + }); + + // Build the guard FIRST so a panicking readiness loop reaps the child. + let guard = TestMinio { + child, + endpoint: format!("http://127.0.0.1:{api_port}"), + _data_dir: data_dir, + }; + let endpoint = guard.endpoint.clone(); + + // Create the test bucket with mc, retrying until the server is up. + // A throwaway --config-dir keeps ~/.mc untouched. + let mc_cfg = tempfile::tempdir().expect("create mc config dir"); + let cfg = mc_cfg.path().to_str().expect("utf-8 mc config path"); + let mut ready = false; + for _ in 0..100 { + let alias = Command::new(&mc_bin) + .args([ + "--config-dir", + cfg, + "alias", + "set", + "t", + &endpoint, + MINIO_USER, + MINIO_PASSWORD, + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap_or_else(|e| panic!("failed to spawn `{mc_bin}`: {e}. Run `mise install`.")); + if alias.success() { + let mb = Command::new(&mc_bin) + .args([ + "--config-dir", + cfg, + "mb", + "--ignore-existing", + &format!("t/{MINIO_BUCKET}"), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("run mc mb"); + if mb.success() { + ready = true; + break; + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + assert!( + ready, + "minio at {endpoint} never became ready (mc mb failed)" + ); + guard + } + + /// Builder options for `ObjectStoreTransport::from_url_opts` pointing at + /// this instance (path-style, http, root credentials). + pub fn s3_options(&self) -> Vec<(&'static str, String)> { + vec![ + ("aws_access_key_id", MINIO_USER.to_string()), + ("aws_secret_access_key", MINIO_PASSWORD.to_string()), + ("aws_endpoint", self.endpoint.clone()), + ("aws_allow_http", "true".to_string()), + ("aws_region", "us-east-1".to_string()), + ] + } +} + +impl Drop for TestMinio { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 8f45940..789f87d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,284 +1,19 @@ //! Shared integration-test infrastructure: throwaway `nats-server` and -//! `minio` instances, one per test, killed on drop. +//! `minio` instances, one per test, killed on drop, and the transport tiers' +//! crash injection. //! //! All binaries come from mise (`mise install`); env overrides //! `NATS_SERVER_BIN` / `MINIO_BIN` / `MC_BIN` point at explicit paths when //! running outside an activated mise shell. No Docker anywhere. -#![allow(dead_code)] // each test crate uses a subset of this harness +#![allow(dead_code, unused_imports)] // each test crate uses a subset of this harness -use std::process::{Child, Command, Stdio}; -use std::time::Duration; +pub mod minio; +pub mod nats; -use tempfile::TempDir; +#[cfg(feature = "transport")] +pub mod crash; -pub fn free_port() -> u16 { - std::net::TcpListener::bind("127.0.0.1:0") - .expect("bind ephemeral port") - .local_addr() - .expect("read local addr") - .port() -} - -// --- NATS ---------------------------------------------------------------------- - -/// A running `nats-server` with JetStream enabled. Killed on drop. -pub struct TestNats { - child: Child, - pub url: String, - _store_dir: TempDir, -} - -impl TestNats { - pub async fn start() -> TestNats { - let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); - let port = free_port(); - let store_dir = tempfile::tempdir().expect("create jetstream store dir"); - let child = Command::new(&bin) - .args([ - "--jetstream", - "--addr", - "127.0.0.1", - "--port", - &port.to_string(), - "--store_dir", - store_dir.path().to_str().expect("utf-8 store path"), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap_or_else(|e| { - panic!( - "failed to spawn `{bin}`: {e}. Is nats-server installed? \ - Run `mise install` or set NATS_SERVER_BIN." - ) - }); - // Build the guard FIRST: if readiness polling panics, Drop reaps the - // child instead of leaking a zombie. - let guard = TestNats { - child, - url: format!("nats://127.0.0.1:{port}"), - _store_dir: store_dir, - }; - for _ in 0..100 { - if async_nats::connect(&guard.url).await.is_ok() { - return guard; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!("nats-server at {} never became ready", guard.url); - } -} - -impl Drop for TestNats { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -// --- MinIO --------------------------------------------------------------------- - -pub const MINIO_USER: &str = "minioadmin"; -pub const MINIO_PASSWORD: &str = "minioadmin"; -pub const MINIO_BUCKET: &str = "test-bucket"; - -/// A running `minio` with [`MINIO_BUCKET`] pre-created via `mc`. Killed on -/// drop. The `mc mb` retry loop doubles as the readiness probe. -pub struct TestMinio { - child: Child, - pub endpoint: String, - _data_dir: TempDir, -} - -impl TestMinio { - pub async fn start() -> TestMinio { - let minio_bin = std::env::var("MINIO_BIN").unwrap_or_else(|_| "minio".to_string()); - let mc_bin = std::env::var("MC_BIN").unwrap_or_else(|_| "mc".to_string()); - let api_port = free_port(); - let console_port = free_port(); - let data_dir = tempfile::tempdir().expect("create minio data dir"); - - let child = Command::new(&minio_bin) - .args([ - "server", - data_dir.path().to_str().expect("utf-8 data path"), - "--address", - &format!("127.0.0.1:{api_port}"), - "--console-address", - &format!("127.0.0.1:{console_port}"), - ]) - .env("MINIO_ROOT_USER", MINIO_USER) - .env("MINIO_ROOT_PASSWORD", MINIO_PASSWORD) - .env("MINIO_BROWSER", "off") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap_or_else(|e| { - panic!( - "failed to spawn `{minio_bin}`: {e}. Is minio installed? \ - Run `mise install` or set MINIO_BIN." - ) - }); - - // Build the guard FIRST so a panicking readiness loop reaps the child. - let guard = TestMinio { - child, - endpoint: format!("http://127.0.0.1:{api_port}"), - _data_dir: data_dir, - }; - let endpoint = guard.endpoint.clone(); - - // Create the test bucket with mc, retrying until the server is up. - // A throwaway --config-dir keeps ~/.mc untouched. - let mc_cfg = tempfile::tempdir().expect("create mc config dir"); - let cfg = mc_cfg.path().to_str().expect("utf-8 mc config path"); - let mut ready = false; - for _ in 0..100 { - let alias = Command::new(&mc_bin) - .args([ - "--config-dir", - cfg, - "alias", - "set", - "t", - &endpoint, - MINIO_USER, - MINIO_PASSWORD, - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .unwrap_or_else(|e| panic!("failed to spawn `{mc_bin}`: {e}. Run `mise install`.")); - if alias.success() { - let mb = Command::new(&mc_bin) - .args([ - "--config-dir", - cfg, - "mb", - "--ignore-existing", - &format!("t/{MINIO_BUCKET}"), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .expect("run mc mb"); - if mb.success() { - ready = true; - break; - } - } - tokio::time::sleep(Duration::from_millis(200)).await; - } - assert!( - ready, - "minio at {endpoint} never became ready (mc mb failed)" - ); - guard - } - - /// Builder options for `ObjectStoreTransport::from_url_opts` pointing at - /// this instance (path-style, http, root credentials). - pub fn s3_options(&self) -> Vec<(&'static str, String)> { - vec![ - ("aws_access_key_id", MINIO_USER.to_string()), - ("aws_secret_access_key", MINIO_PASSWORD.to_string()), - ("aws_endpoint", self.endpoint.clone()), - ("aws_allow_http", "true".to_string()), - ("aws_region", "us-east-1".to_string()), - ] - } -} - -impl Drop for TestMinio { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -// --- Crash injection for the transport tiers --------------------------------- - -/// An [`object_store::ObjectStore`] that, once armed, fails the next PUT of a -/// `*.manifest.json` object — the process crash in the window between -/// `upload`'s payload multipart (already complete) and its sibling-manifest -/// PUT. Everything else delegates to the wrapped store. -#[derive(Debug)] -pub struct ManifestPutCrash { - pub inner: std::sync::Arc, - pub armed: std::sync::atomic::AtomicBool, -} - -impl std::fmt::Display for ManifestPutCrash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "ManifestPutCrash({})", self.inner) - } -} - -#[async_trait::async_trait] -impl object_store::ObjectStore for ManifestPutCrash { - async fn put_opts( - &self, - location: &object_store::path::Path, - payload: object_store::PutPayload, - opts: object_store::PutOptions, - ) -> object_store::Result { - if location.as_ref().ends_with(".manifest.json") - && self.armed.swap(false, std::sync::atomic::Ordering::SeqCst) - { - return Err(object_store::Error::Generic { - store: "ManifestPutCrash", - source: "injected crash between payload and manifest PUT".into(), - }); - } - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &object_store::path::Path, - opts: object_store::PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts( - &self, - location: &object_store::path::Path, - options: object_store::GetOptions, - ) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: futures::stream::BoxStream< - 'static, - object_store::Result, - >, - ) -> futures::stream::BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list( - &self, - prefix: Option<&object_store::path::Path>, - ) -> futures::stream::BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter( - &self, - prefix: Option<&object_store::path::Path>, - ) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts( - &self, - from: &object_store::path::Path, - to: &object_store::path::Path, - options: object_store::CopyOptions, - ) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } -} +#[cfg(feature = "transport")] +pub use crash::ManifestPutCrash; +pub use minio::{MINIO_BUCKET, MINIO_PASSWORD, MINIO_USER, TestMinio}; +pub use nats::{TestNats, free_port}; diff --git a/tests/common/nats.rs b/tests/common/nats.rs new file mode 100644 index 0000000..9e957fd --- /dev/null +++ b/tests/common/nats.rs @@ -0,0 +1,76 @@ +//! A throwaway `nats-server` with JetStream, one per test, killed on drop. +//! +//! THE one copy: the integration tests reach it through `tests/common`, and +//! the crate's own unit tests through `#[path]` (`src/nats.rs`), so it +//! depends only on crates both can see (`async-nats`, `tempfile`, `tokio`). +//! `nats-server` comes from mise (`mise install`); `NATS_SERVER_BIN` points +//! at an explicit path outside an activated mise shell. +#![allow(dead_code)] // each test crate uses a subset of this harness + +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use tempfile::TempDir; + +pub fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind ephemeral port") + .local_addr() + .expect("read local addr") + .port() +} + +/// A running `nats-server` with JetStream enabled. Killed on drop. +pub struct TestNats { + child: Child, + pub url: String, + _store_dir: TempDir, +} + +impl TestNats { + pub async fn start() -> TestNats { + let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); + let port = free_port(); + let store_dir = tempfile::tempdir().expect("create jetstream store dir"); + let child = Command::new(&bin) + .args([ + "--jetstream", + "--addr", + "127.0.0.1", + "--port", + &port.to_string(), + "--store_dir", + store_dir.path().to_str().expect("utf-8 store path"), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap_or_else(|e| { + panic!( + "failed to spawn `{bin}`: {e}. Is nats-server installed? \ + Run `mise install` or set NATS_SERVER_BIN." + ) + }); + // Build the guard FIRST: if readiness polling panics, Drop reaps the + // child instead of leaking a zombie. + let guard = TestNats { + child, + url: format!("nats://127.0.0.1:{port}"), + _store_dir: store_dir, + }; + for _ in 0..100 { + if async_nats::connect(&guard.url).await.is_ok() { + return guard; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("nats-server at {} never became ready", guard.url); + } +} + +impl Drop for TestNats { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} diff --git a/tests/dst_invariants.rs b/tests/dst_invariants.rs index 6fd4cce..95a0dc2 100644 --- a/tests/dst_invariants.rs +++ b/tests/dst_invariants.rs @@ -2,7 +2,7 @@ //! //! These assert the *correct* behavior. They fail on current main (B1/B3/B4). -use slipstream::snapshot::{load, SnapshotError, SnapshotWriter}; +use slipstream::snapshot::{SnapshotError, SnapshotWriter, load}; use slipstream::{AppendLogSnapshot, KvEntry, KvUpdate, SnapshotStore, VersionToken, WatchCursor}; use std::path::Path; use tempfile::TempDir; diff --git a/tests/integration.rs b/tests/integration.rs index 6ee0290..85064e8 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -2,16 +2,17 @@ //! //! Each test boots its own `nats-server` (JetStream enabled) on a free port with //! a throwaway store directory, then talks to it through the public `slipstream` -//! API. The server is killed when the [`TestNats`] guard drops, so tests are -//! fully isolated and leave nothing running. +//! API (the shared harness in `tests/common`). The server is killed when the +//! [`TestNats`] guard drops, so tests are fully isolated and leave nothing +//! running. //! //! `nats-server` comes from mise (`ubi:nats-io/nats-server`). When mise is //! activated it's on `PATH`; otherwise set `NATS_SERVER_BIN` to an explicit path. -use std::process::{Child, Command, Stdio}; use std::sync::Arc; use std::time::Duration; +use common::TestNats; use slipstream::{ Connection, DiscardPolicy, KvError, KvStore, KvUpdate, KvWriter, NatsConnection, NatsConnectionConfig, StoreConfig, VersionToken, WatchCursor, @@ -19,55 +20,21 @@ use slipstream::{ use tokio::sync::mpsc; use tokio::time::timeout; +mod common; + // --- Test harness ------------------------------------------------------------ -/// A running `nats-server` with JetStream enabled. Killed on drop. -struct TestNats { - child: Child, - url: String, - // Kept alive so the JetStream store directory survives for the server's - // lifetime; removed when the guard drops. - _store_dir: tempfile::TempDir, -} +/// The public API on a [`TestNats`] server. +trait PublicApi { + /// Connect through the public API and return a ready connection. + async fn connect(&self) -> NatsConnection; -impl TestNats { - /// Boot a fresh server and block until it accepts connections. - async fn start() -> TestNats { - let bin = std::env::var("NATS_SERVER_BIN").unwrap_or_else(|_| "nats-server".to_string()); - let port = free_port(); - let store_dir = tempfile::tempdir().expect("create jetstream store dir"); - - let child = Command::new(&bin) - .args([ - "--jetstream", - "--addr", - "127.0.0.1", - "--port", - &port.to_string(), - "--store_dir", - store_dir.path().to_str().expect("utf-8 store path"), - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap_or_else(|e| { - panic!( - "failed to spawn `{bin}`: {e}. Is nats-server installed? \ - Run `mise install` or set NATS_SERVER_BIN." - ) - }); - - let url = format!("nats://127.0.0.1:{port}"); - wait_until_ready(&url).await; - - TestNats { - child, - url, - _store_dir: store_dir, - } - } + /// Connect and open a store named `bucket`. The connection is returned + /// too because it owns the underlying NATS client; keep it in scope. + async fn store(&self, bucket: &str) -> (NatsConnection, Arc); +} - /// Connect through the public API and return a ready connection. +impl PublicApi for TestNats { async fn connect(&self) -> NatsConnection { let conn = NatsConnection::new(NatsConnectionConfig { url: self.url.clone(), @@ -78,8 +45,6 @@ impl TestNats { conn } - /// Connect and open a store named `bucket`. The connection is returned too - /// because it owns the underlying NATS client; keep it in scope. async fn store(&self, bucket: &str) -> (NatsConnection, Arc) { let conn = self.connect().await; let store = conn @@ -94,33 +59,6 @@ impl TestNats { } } -impl Drop for TestNats { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -/// Grab a free TCP port by binding to :0 and reading the assigned port back. -fn free_port() -> u16 { - std::net::TcpListener::bind("127.0.0.1:0") - .expect("bind ephemeral port") - .local_addr() - .expect("read local addr") - .port() -} - -/// Poll the server until a client connects or we give up. -async fn wait_until_ready(url: &str) { - for _ in 0..100 { - if async_nats::connect(url).await.is_ok() { - return; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!("nats-server at {url} never became ready"); -} - /// Deterministically wait until a freshly spawned watch is live. /// /// Watches deliver the current state first (the re-list), then live updates — diff --git a/tests/model.rs b/tests/model.rs index 9fa223a..434129d 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -764,25 +764,40 @@ impl Model for SnapshotProtocol { // markers were evicted are lost and the resync never // triggers, regardless of the resync mode. s.importer = ImporterPc::Resumed(a, Self::relist_only_status(&s, a)); - } else if self.resync == ResyncMode::None { - s.importer = ImporterPc::Resumed(a, Self::relist_only_status(&s, a)); - } else if self.retention == BucketRetention::EvictsCurrent - && self.mutation != Mutation::RelistOnEvicting - { - // CursorExpired on a bucket that evicts current values: - // NATS's listing can't separate deleted from aged out, - // so the fold is restored from an artifact - // (`ExpiryRepair::Auto` → Restore). - s.importer = ImporterPc::Restoring(a); } else { - // CursorExpired -> full re-list + a SUCCESSFUL key- - // listing resync: live keys diffed against the fold, - // vanished keys get synthetic deletes. (Under FailStop a - // failed resync fails the watch and changes nothing — - // this action stays enabled for the retry. Under Degrade - // the failed-resync outcome is ResumeResyncDegraded.) - // Sound exactly when the listing is the truth. - s.importer = ImporterPc::Resumed(a, Self::relist_status(&s)); + // CursorExpired: the shared planner decides, under the + // shipped `ExpiryRepair::Auto` (or `None`, no repair). + // `RelistOnEvicting` forges the listing as the truth. + use slipstream::protocol::{ + RepairMode, RepairPlan, listing_is_truth, plan_repair, + }; + let mode = match self.resync { + ResyncMode::None => RepairMode::None, + ResyncMode::FailStop | ResyncMode::Degrade => RepairMode::Auto, + }; + let truth = self.mutation == Mutation::RelistOnEvicting + || listing_is_truth( + self.retention == BucketRetention::EvictsCurrent, + s.floor as u64 + 1, + ); + s.importer = match plan_repair(mode, Some(truth)) { + RepairPlan::ReListOnly => { + ImporterPc::Resumed(a, Self::relist_only_status(&s, a)) + } + // On a bucket that evicts current values, NATS's + // listing can't separate deleted from aged out, so + // the fold is restored from an artifact. + RepairPlan::Restore => ImporterPc::Restoring(a), + // Full re-list + a SUCCESSFUL key-listing resync: + // live keys diffed against the fold, vanished keys + // get synthetic deletes. (Under FailStop a failed + // resync fails the watch and changes nothing — this + // action stays enabled for the retry. Under Degrade + // the failed-resync outcome is ResumeResyncDegraded.) + // Sound exactly when the listing is the truth. + RepairPlan::Relist => ImporterPc::Resumed(a, Self::relist_status(&s)), + RepairPlan::RefuseRelist => unreachable!("Auto never refuses"), + }; } } Act::RestoreRead => { diff --git a/tests/model_fleet.rs b/tests/model_fleet.rs index 393f2d3..44fac85 100644 --- a/tests/model_fleet.rs +++ b/tests/model_fleet.rs @@ -33,7 +33,8 @@ //! each poisons the fleet's artifacts. use slipstream::protocol::{ - PointerState, pointer_publish_allowed, restore_allowed, resume_window_ok, + PointerState, RepairMode, RepairPlan, cursorless_start_needs_repair, listing_is_truth, + plan_repair, pointer_publish_allowed, restore_allowed, resume_window_ok, }; use stateright::{Checker, Model, Property}; @@ -109,16 +110,26 @@ impl Fleet { s.head as u64 } } - fn listing_is_truth(s: &St) -> bool { - resume_window_ok(0, Self::first_revision(s)) - } fn listed(s: &St) -> Option { if s.evicted { None } else { Self::truth(s) } } - /// The repair a node would take: the artifact restore, or the - /// key-listing diff (shipped: `ExpiryRepair::Auto`). + /// The listing as the planner sees it: the shared kernel on the bucket's + /// retention. `RelistOnEvicting` forges it as the truth. + fn listing_truth(&self, s: &St) -> Option { + if self.mutation == Mutation::RelistOnEvicting { + return Some(true); + } + Some(listing_is_truth(self.evicting, Self::first_revision(s))) + } + /// Does an expired node repair by the artifact restore (else the + /// key-listing diff)? The shared planner, under the shipped + /// `ExpiryRepair::Auto`. fn restores(&self, s: &St) -> bool { - self.evicting && self.mutation != Mutation::RelistOnEvicting && !Self::listing_is_truth(s) + match plan_repair(RepairMode::Auto, self.listing_truth(s)) { + RepairPlan::Restore => true, + RepairPlan::Relist => false, + plan => unreachable!("Auto never plans {plan:?}"), + } } fn restore_target(s: &St, n: &Node) -> Option<(u8, Option)> { let (c, v) = s.pointer?; @@ -212,7 +223,14 @@ impl Model for Fleet { } Act::Evict => s.evicted = true, Act::Start(i) => { - let relist = self.mutation == Mutation::FreshStartRelist || !self.restores(&s); + // The shared start kernel on an empty fold. `FreshStartRelist` + // forges the listing as the truth. + let truth = if self.mutation == Mutation::FreshStartRelist { + Some(true) + } else { + self.listing_truth(&s) + }; + let relist = !cursorless_start_needs_repair(RepairMode::Auto, false, truth); let (listed, head) = (Fleet::listed(&s), s.head); let n = &mut s.nodes[i]; n.fresh = false; diff --git a/tests/model_live_watch.rs b/tests/model_live_watch.rs index dbbe925..e006396 100644 --- a/tests/model_live_watch.rs +++ b/tests/model_live_watch.rs @@ -69,7 +69,9 @@ //! bootstrapped fold). `Publish` may lag arbitrarily — a stale pointer is //! exactly what the restore guard must refuse. -use slipstream::protocol::{restore_allowed, resume_window_ok}; +use slipstream::protocol::{ + RepairMode, RepairPlan, listing_is_truth, plan_repair, restore_allowed, resume_window_ok, +}; use stateright::{Checker, Model, Property}; /// Stream revisions run 1..=MAX_REV. @@ -179,11 +181,31 @@ impl LiveWatch { Self::truth(s).filter(|_| !s.aged) } - fn restores(&self) -> bool { + /// Publishing only matters where a repair could restore (a state-space + /// cut, not a decision: the planner below decides). + fn artifacts_matter(&self) -> bool { self.retention == BucketRetention::EvictsCurrent && self.mutation != Mutation::RelistOnEvicting } + /// Does a trip repair by the artifact restore (else the key-listing + /// diff)? The shared planner under the shipped `ExpiryRepair::Auto`, on + /// the listing as the kernel reads the bucket's retention (first + /// retained revision `floor + 1`). `RelistOnEvicting` forges the listing + /// as the truth. + fn restores(&self, s: &St) -> bool { + let truth = self.mutation == Mutation::RelistOnEvicting + || listing_is_truth( + self.retention == BucketRetention::EvictsCurrent, + s.floor as u64 + 1, + ); + match plan_repair(RepairMode::Auto, Some(truth)) { + RepairPlan::Restore => true, + RepairPlan::Relist => false, + plan => unreachable!("Auto never plans {plan:?}"), + } + } + /// The restore this repair may take from the current pointer, if any. fn restore_target(&self, s: &St) -> Option { let p = s.pointer?; @@ -234,7 +256,7 @@ impl Model for LiveWatch { acts.push(Act::Compact); } // Artifacts only matter where a restore can use them. - if self.restores() { + if self.artifacts_matter() { for c in s.pointer.map_or(1, |p| p + 1)..=s.head { acts.push(Act::Publish(c)); } @@ -260,7 +282,7 @@ impl Model for LiveWatch { // fail-stop until the fleet publishes one. if self.guarded && !resume_window_ok(s.frontier as u64, s.floor as u64 + 1) - && (!self.restores() || self.restore_target(s).is_some()) + && (!self.restores(s) || self.restore_target(s).is_some()) { acts.push(Act::GuardRepair); } @@ -301,7 +323,7 @@ impl Model for LiveWatch { Act::GuardRepair => { s.tripped = true; let before = s.frontier; - if self.restores() { + if self.restores(&s) { // Artifact restore: the fold becomes the artifact's, // consumption resumes from its cursor. let p = self.restore_target(&s)?; @@ -365,7 +387,7 @@ impl Model for LiveWatch { "the floor guard trips and repairs a real divergence", |_, s| s.tripped && s.fold_key == LiveWatch::truth(s), )); - if self.restores() { + if self.artifacts_matter() { props.push(Property::::sometimes( "a trip restores from an artifact past a value that aged out", |_, s| s.restored && s.aged && s.fold_key == LiveWatch::truth(s), diff --git a/tests/model_repair.rs b/tests/model_repair.rs index e7f91cb..8c4461f 100644 --- a/tests/model_repair.rs +++ b/tests/model_repair.rs @@ -48,7 +48,10 @@ //! discharged by `tests/model_fleet.rs`); a full re-list is not overrun by //! retention before it delivers what it found (axiom 6 of `tests/model.rs`). -use slipstream::protocol::{restore_allowed, resume_window_ok}; +use slipstream::protocol::{ + KeyRestore, KeyState, RepairMode, RepairPlan, cursorless_start_needs_repair, listing_is_truth, + plan_repair, restore_ahead, restore_allowed, restore_key, resume_window_ok, +}; use stateright::{Checker, Model, Property}; /// Capacity of the revision array (bounds are per model: `max_rev`). @@ -207,6 +210,8 @@ enum Mutation { #[derive(Clone)] struct RepairModel { evicting: bool, + /// The repair armed (`ExpiryRepair`'s mode). + mode: RepairMode, fresh: bool, mutation: Mutation, /// Let retention evict during a key-listing repair (see `actions`). @@ -239,10 +244,13 @@ impl RepairModel { } } - /// `listing_is_truth` in `applied.rs`: retention never evicts current - /// values, or has never evicted anything yet. - fn listing_is_truth(s: &St) -> bool { - resume_window_ok(0, Self::first_revision(s)) + /// The listing as the planner sees it: the shared kernel on the bucket's + /// retention. `RelistOnEvicting` forges it as the truth. + fn listing_truth(&self, s: &St) -> Option { + if self.mutation == Mutation::RelistOnEvicting { + return Some(true); + } + Some(listing_is_truth(self.evicting, Self::first_revision(s))) } fn listed(s: &St) -> Option { @@ -371,35 +379,32 @@ impl RepairModel { } } - /// `restore_diff` on the key: the artifact at `target` holds the truth - /// there. Take it unless the store's value is the same or NEWER; delete - /// (the artifact lacks the key) unless the listing shows it live. + /// `restore_diff` on the key, through the shared kernel: the artifact at + /// `target` holds the truth there. `RestoreIgnoresVersions` forges both + /// sides revisionless (the pre-fix diff: take anything not identical); + /// `RestoreIgnoresListing` forges the key unlisted. fn restore_op(&self, s: &St, target: u8, listed: Option) -> Option { let want = Self::truth_at(s, target); - match (s.store_val, want) { - (Some(l), Some(a)) if l >= a && self.mutation != Mutation::RestoreIgnoresVersions => { - None - } - (_, Some(a)) if s.store_val != Some(a) => Some(Upd { + let versions = self.mutation != Mutation::RestoreIgnoresVersions; + let state = |v: Option| match v { + Some(r) => KeyState::At(versions.then_some(r as u64)), + None => KeyState::Absent, + }; + let listed_live = listed.is_some() && self.mutation != Mutation::RestoreIgnoresListing; + let identical = s.store_val.is_some() && s.store_val == want; + match restore_key(state(s.store_val), state(want), identical, listed_live) { + KeyRestore::Keep => None, + KeyRestore::TakeArtifact => Some(Upd { pos: None, - value: Some(a), + value: want, + }), + KeyRestore::Delete => Some(Upd { + pos: None, + value: None, }), - (Some(_), None) - if listed.is_none() || self.mutation == Mutation::RestoreIgnoresListing => - { - Some(Upd { - pos: None, - value: None, - }) - } - _ => None, } } - fn restores(&self) -> bool { - self.evicting && self.mutation != Mutation::RelistOnEvicting - } - /// Resume from cursor `c` (> 0): the window check. fn resume(s: &St, c: u8) -> WPhase { if resume_window_ok(c as u64, Self::first_revision(s)) { @@ -587,14 +592,13 @@ impl Model for RepairModel { Act::Publish(c) => s.pointer = c, Act::WStart => { s.w = if s.store_cur == 0 { - if s.store_val.is_some() && self.mutation != Mutation::UnanchoredRelist { - // Data but no cursor (a torn first checkpoint): a - // re-list alone never removes what it doesn't - // deliver — repair like an expiry at 0. - WPhase::Expired { local: 0 } - } else if self.restores() && !Self::listing_is_truth(&s) { - // Empty fold, bucket already evicted current values: - // seed from an artifact. + // Data but no cursor (a torn first checkpoint), or an + // empty fold on a bucket that already evicted current + // values: repair like an expiry at 0. `UnanchoredRelist` + // forges the fold empty. + let has_data = + s.store_val.is_some() && self.mutation != Mutation::UnanchoredRelist; + if cursorless_start_needs_repair(self.mode, has_data, self.listing_truth(&s)) { WPhase::Expired { local: 0 } } else { WPhase::Watch { @@ -626,31 +630,41 @@ impl Model for RepairModel { let WPhase::Expired { local } = s.w else { return None; }; - if self.restores() && !Self::listing_is_truth(&s) { - // Peek + fetch collapsed onto the CURRENT pointer (the - // code re-checks what it fetched against now). - let target = s.pointer; - if self.restore_ok(&s, target, local) && target > 0 { - s.req = Some(Req::Restore { - target, + match plan_repair(self.mode, self.listing_truth(&s)) { + RepairPlan::Restore => { + // Peek + fetch collapsed onto the CURRENT pointer + // (the code re-checks what it fetched against now). + let target = s.pointer; + if self.restore_ok(&s, target, local) && target > 0 { + s.req = Some(Req::Restore { + target, + listed: Self::listed(&s), + }); + s.w = WPhase::AwaitReply; + } else { + s.w = WPhase::Failed; + } + } + RepairPlan::Relist => { + s.req = Some(Req::Relist { listed: Self::listed(&s), }); - s.w = WPhase::AwaitReply; - } else { - s.w = WPhase::Failed; + s.w = if self.mutation == Mutation::NoAckBarrier { + WPhase::Watch { + f: 0, + guarded: false, + } + } else { + WPhase::AwaitAck + }; } - } else { - s.req = Some(Req::Relist { - listed: Self::listed(&s), - }); - s.w = if self.mutation == Mutation::NoAckBarrier { - WPhase::Watch { + RepairPlan::RefuseRelist => s.w = WPhase::Failed, + RepairPlan::ReListOnly => { + s.w = WPhase::Watch { f: 0, guarded: false, } - } else { - WPhase::AwaitAck - }; + } } } Act::WGotReply => { @@ -716,7 +730,9 @@ impl Model for RepairModel { } MPhase::PreFlush(Req::Restore { target, listed }, _) => { // The authoritative ahead check. - if self.mutation != Mutation::NoRestoreGuard && target <= s.applied { + if self.mutation != Mutation::NoRestoreGuard + && !restore_ahead(target as u64, s.applied as u64) + { s.reply = Some(Reply::Refused); s.m = MPhase::Idle; } else if self.mutation == Mutation::CursorBeforeDiff { @@ -792,7 +808,7 @@ impl Model for RepairModel { if self.mutation == Mutation::None { // The theorems are earned: the repairs run, the hazards they // guard against occur. - if self.restores() { + if self.evicting { props.push(Property::::sometimes( "a restore completes past an evicted current value", |_, s| s.restored && s.evicted && RepairModel::truth(s).is_some(), @@ -826,9 +842,16 @@ fn run(model: RepairModel, label: &str) -> impl Checker { checker } +/// The shipped configuration: `ExpiryRepair::Auto` on an evicting bucket, +/// `ExpiryRepair::Relist` on one that keeps current values. fn shipped(evicting: bool, fresh: bool) -> RepairModel { RepairModel { evicting, + mode: if evicting { + RepairMode::Auto + } else { + RepairMode::Relist + }, fresh, mutation: Mutation::None, drop_axiom_6: false, @@ -841,6 +864,14 @@ fn shipped(evicting: bool, fresh: bool) -> RepairModel { #[test] fn keeps_current_repair_steps_are_correct() { run(shipped(false, false), "repair steps: keeps-current").assert_properties(); + run( + RepairModel { + mode: RepairMode::Auto, + ..shipped(false, false) + }, + "repair steps: keeps-current, Auto", + ) + .assert_properties(); } #[test] diff --git a/tests/snapshot_store.rs b/tests/snapshot_store.rs index 01d911a..13a59c2 100644 --- a/tests/snapshot_store.rs +++ b/tests/snapshot_store.rs @@ -192,6 +192,29 @@ fn check_for_each_in_range(open: impl Fn(&Path) -> (WatchCurso assert_eq!(seen, 2, "scan stops at the first callback error"); } +/// `has_entries` is "the fold holds any live entry": false empty, true once a +/// key lands, false again once every key is deleted or purged, and the same +/// after a reopen. +fn check_has_entries(open: impl Fn(&Path) -> (WatchCursor, S)) { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store"); + let (_resume, mut s) = open(&path); + assert!(!s.has_entries().unwrap(), "a new fold is empty"); + fold(&mut s, &[put("a", b"1", 1), put("b", b"2", 2)]); + assert!(s.has_entries().unwrap()); + fold(&mut s, &[del("a", 3)]); + assert!(s.has_entries().unwrap(), "one key left"); + fold(&mut s, &[purge("b", 4)]); + assert!(!s.has_entries().unwrap(), "every key deleted or purged"); + drop(s); + let (_resume, mut s) = open(&path); + assert!(!s.has_entries().unwrap(), "still empty after a reopen"); + fold(&mut s, &[put("c", b"3", 5)]); + drop(s); + let (_resume, s) = open(&path); + assert!(s.has_entries().unwrap(), "an entry survives a reopen"); +} + /// Cursor-resume after a reconnect: fold a first segment, reopen (cursor reflects /// it), fold the post-cursor delta, reopen again (cursor advanced). Models a /// service restarting and resuming the watch from the persisted position. @@ -713,6 +736,11 @@ fn append_log_for_each_in_range() { check_for_each_in_range(open_append_log); } +#[test] +fn append_log_has_entries() { + check_has_entries(open_append_log); +} + #[test] fn append_log_cursor_resume() { check_cursor_resume(open_append_log); @@ -867,6 +895,11 @@ mod fjall_backend { check_for_each_in_range(open_no_sync); } + #[test] + fn fjall_has_entries() { + check_has_entries(open_no_sync); + } + #[test] fn fjall_cursor_resume() { check_cursor_resume(open_no_sync); @@ -1085,6 +1118,11 @@ mod rocksdb_backend { check_for_each_in_range(open_no_sync); } + #[test] + fn rocksdb_has_entries() { + check_has_entries(open_no_sync); + } + #[test] fn rocksdb_cursor_resume() { check_cursor_resume(open_no_sync);