diff --git a/docs/src/design/orchestrator/orchestrator-machine.md b/docs/src/design/orchestrator/orchestrator-machine.md index b7227f8a..69c9dfad 100644 --- a/docs/src/design/orchestrator/orchestrator-machine.md +++ b/docs/src/design/orchestrator/orchestrator-machine.md @@ -35,6 +35,8 @@ stateDiagram-v2 Recovering --> PreSupervision : Restored [retry < max_retry]
(re-verify) Recovering --> PreSupervision : Restored [retry ≥ max_retry, Isolable/Cascading]
/ AssertReset (skip — held) Recovering --> Locked : Restored [retry ≥ max_retry, Required]
(self-emits RecoveryFailed) / LatchLockdown + Recovering --> PreSupervision : RecoveryUnavailable [Isolable/Cascading]
/ AssertReset (skip — held) + Recovering --> Locked : RecoveryUnavailable [Required]
(self-emits RecoveryFailed) / LatchLockdown Locked --> Locked : (terminal — all events ignored) ``` @@ -232,11 +234,26 @@ entire region is affected (not the whole chain — INV5). | `Restored(id)` | `id == failed`, cap reached, `Isolable` | `AssertReset(failed)` · `ReportIsolated(failed)` | `PreSupervision` (recovery exhausted: mark `failed` `Isolated`, reset its `retry`; the re-walk skips it) | | `Restored(id)` | `id == failed`, cap reached, `Cascading` | `AssertReset` · `ReportIsolated` (per component) | `PreSupervision` (recovery exhausted: mark `failed` + `depends_on` dependents `Isolated`) | | `Restored(id)` | `id == failed`, cap reached, `Required` | `ReportRecoveryFailed(failed)` · `Effect::Emit(RecoveryFailed)` | `Handled` (orchestrator queues `RecoveryFailed` next — INV7) | +| `RecoveryUnavailable(id)` | `id != failed` | — | `Handled` (stale — belongs to a displaced episode) | +| `RecoveryUnavailable(id)` | `id == failed`, `Isolable` | `AssertReset(failed)` · `ReportIsolated(failed)` | `PreSupervision` (exhausted immediately, bypassing the retry cap) | +| `RecoveryUnavailable(id)` | `id == failed`, `Cascading` | `AssertReset` · `ReportIsolated` (per component) | `PreSupervision` (exhausted immediately) | +| `RecoveryUnavailable(id)` | `id == failed`, `Required` | `ReportRecoveryFailed(failed)` · `Effect::Emit(RecoveryFailed)` | `Handled` | | `RecoveryFailed` | — | — | `Locked` | | anything else | — | — | `Outcome::Super` → `SupervisingPlatform` | ("cap reached" = `retry + 1 >= max_retry`.) +**`RecoveryUnavailable` is the platform's own exhaustion signal.** Where +`Restored` past the retry cap means "the machine gave up after counting failed +restores," `RecoveryUnavailable` means "the platform driver reports no +recovery source left for this component" — reported in place of `Restored` +when it has no untried image/slot to swap in. It is authoritative: it runs +the same exhaustion handling as the retry cap (`Isolable`/`Cascading` gate and +skip, `Required` locks) immediately, without bumping or waiting on `retry`. +This keeps `max_retry` a pure liveness backstop against a restore that keeps +"succeeding" without ever re-verifying clean, rather than the only way +recovery ever ends. + **Two-stage recovery (CSA-aligned).** A verification failure never skips a component outright. Every failure — during initial boot or a re-walk — first brings the machine here, to `Recovering`, which restores the failed component's diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index 608cb60f..e049c816 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs @@ -440,6 +440,28 @@ impl Rot { } } + /// Recovery is over for `failed` without success: gate per policy + /// (`Isolable`/`Cascading` skip; `Required` reports + latches `Locked`). + /// Shared by the retry-cap path (`Restored`, count exhausted) and the + /// platform's `RecoveryUnavailable` path, so the two can never diverge. + fn exhaust_recovery(&mut self, ctx: &mut Sink, failed: ComponentId) -> Outcome { + match self.gate_by_policy(ctx, failed) { + Gating::Gated => { + self.clear_retry(failed); + Outcome::Transition(State::PreSupervision) + } + // `Required`, or an unknown/missing id: report the component that + // forced the halt, then lock down. The report precedes the + // internal `Emit`, so it is actuated before the machine moves + // toward `Locked`. + Gating::NotGated => { + ctx.emit(Effect::ReportRecoveryFailed(failed)); + ctx.emit(Effect::Emit(Event::RecoveryFailed)); + Outcome::Handled + } + } + } + /// Shared `CorruptionDetected` handling. Delegates the policy interpretation /// to [`gate_by_policy`](Self::gate_by_policy) so this path and the /// recovery-exhaustion path can never diverge: @@ -774,26 +796,20 @@ impl Rot { if attempts < self.max_retry { Outcome::Transition(State::PreSupervision) } else { - // Retries exhausted: gate via the same `gate_by_policy` - // the runtime-corruption path uses, so the two can never - // disagree. Gated → continue the walk; NotGated - // (Required/unknown) → lock down. - match self.gate_by_policy(ctx, failed) { - Gating::Gated => { - self.clear_retry(failed); - Outcome::Transition(State::PreSupervision) - } - // `Required`, or an unknown/missing id: report the - // component that forced the halt, then lock down. - // The report precedes the internal `Emit`, so it is - // actuated before the machine moves toward `Locked`. - Gating::NotGated => { - ctx.emit(Effect::ReportRecoveryFailed(failed)); - ctx.emit(Effect::Emit(Event::RecoveryFailed)); - Outcome::Handled - } - } + // Retries exhausted: gate via the same shared arm the + // platform's `RecoveryUnavailable` path uses, so the + // two can never disagree. + self.exhaust_recovery(ctx, failed) + } + } + Event::RecoveryUnavailable(id) => { + if *id != failed { + return Outcome::Handled; // same guard as Restored } + // Authoritative: the platform is out of sources, so the + // machine does not wait for the retry count to run out + // (does not call `bump_retry`). + self.exhaust_recovery(ctx, failed) } Event::RecoveryFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, @@ -992,6 +1008,16 @@ pub struct EffectError; /// of the escalation ladder — the core has nothing stronger to emit and /// will *believe* it is `Locked`. The driver must treat that failure as /// terminal (halt/reset), not a recoverable error. +/// - **[`Effect::RecoverComponent`] reports its verdict as an event, not an +/// `execute` error.** On success the driver feeds back +/// [`Event::Restored`]; when its configured recovery sources for that +/// component are exhausted, it feeds back [`Event::RecoveryUnavailable`] +/// instead — never [`EffectError`]. `EffectError` from a `RecoverComponent` +/// call is reserved for a genuine actuation fault (e.g. a bus error during +/// the image swap), which fails closed to [`State::Locked`] unconditionally. +/// Reporting "out of images" that way would lock the whole platform down +/// even for an `Isolable`/`Cascading` component, instead of letting it be +/// gated per [`FailurePolicy`] like the count-driven exhaustion path. pub trait Platform { fn execute(&mut self, effect: Effect) -> Result, EffectError>; } diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index 64ac92b6..f7458e5f 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs @@ -219,6 +219,11 @@ pub enum Event { CorruptionDetected(ComponentId), /// This component has been restored from its configured recovery source. Restored(ComponentId), + /// The platform has no remaining recovery source for `id` (its configured + /// images/slots are exhausted); reported in place of `Restored(id)` and + /// short-circuits the retry cap. See the verdict-vs-error contract on + /// [`Platform::execute`](crate::Platform::execute). + RecoveryUnavailable(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, /// The platform driver's boot-progress watchdog fired: `id` did not report its @@ -267,6 +272,7 @@ impl Event { | Event::BootConfirmed(id) | Event::CorruptionDetected(id) | Event::Restored(id) + | Event::RecoveryUnavailable(id) | Event::Timeout(id) => Some(*id), Event::PowerGood(_) | Event::AttestationChallenge @@ -320,6 +326,10 @@ pub enum Effect { /// attempt 0, slot B on 1, golden on 2) without counting attempts itself — /// a count of its own could drift from the core's, since the driver never /// sees when a recovery succeeds. + /// + /// The driver reports the verdict as an event, not an `execute` error — + /// see the verdict-vs-error contract on + /// [`Platform::execute`](crate::Platform::execute). RecoverComponent { id: ComponentId, attempt: u8, diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index b62a2e14..21425d17 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs @@ -1388,6 +1388,151 @@ fn required_exhaustion_reports_before_lockdown() { ); } +/// Platform-signaled recovery exhaustion (`RecoveryUnavailable`) short-circuits +/// the retry cap: a single event, not `MAX_RETRY` cycles, is enough for an +/// `Isolable` component to be gated and skipped. +#[test] +fn isolable_recovery_unavailable_skips() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C1), // platform: no image left + Event::VerificationPassed(C0), // re-walk from top + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(effects.contains(&Effect::ReportIsolated(C1))); + assert!(!effects.contains(&Effect::ReleaseReset(C1))); // never released + assert!(!effects.contains(&Effect::LatchLockdown)); // NOT a lockdown +} + +/// Platform-signaled recovery exhaustion on a `Required` component reports it +/// then latches `Locked` — no lockdown-avoidance policy applies to `Required`. +#[test] +fn required_recovery_unavailable_locks() { + let (effects, state) = drive( + passive_required(&[C0]), + &[ + BOOT, + Event::VerificationFailed(C0), // → Recovering(C0) + Event::RecoveryUnavailable(C0), + ], + ); + assert_eq!(state, State::Locked); + assert!(effects.contains(&Effect::ReportRecoveryFailed(C0))); + assert!(effects.contains(&Effect::LatchLockdown)); +} + +/// A `Cascading` root that goes `RecoveryUnavailable` gates itself and its +/// transitive dependents, exactly like count-driven cascade exhaustion. +#[test] +fn cascading_recovery_unavailable_cascades() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_cascading()), + (C2, ComponentAttrs::passive_required().with_depends_on(C1)), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C1), + Event::VerificationPassed(C0), // re-walk from top + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::AssertReset(C1))); + assert!(effects.contains(&Effect::AssertReset(C2))); + assert!(effects.contains(&Effect::ReportIsolated(C1))); + assert!(effects.contains(&Effect::ReportIsolated(C2))); + assert!(!effects.contains(&Effect::LatchLockdown)); +} + +/// `RecoveryUnavailable` is authoritative and immediate: it exhausts recovery +/// on the first event regardless of how high `max_retry` is, and never +/// consults (or bumps) the retry count. +#[test] +fn recovery_unavailable_short_circuits_retry_budget() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_isolable())) + .expect("fits"); + c.push((C1, ComponentAttrs::passive_required())) + .expect("fits"); + let mut orch = Orchestrator::::new(c.try_into().expect("valid chain"), u8::MAX); + let mut effects = Vec::new(); + for ev in [ + BOOT, + Event::VerificationFailed(C0), // → Recovering(C0) + Event::RecoveryUnavailable(C0), + Event::VerificationPassed(C1), // re-walk skips gated C0, reaches Ready + ] { + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(None) + }); + } + assert_eq!(orch.state(), State::Ready); + assert_eq!( + effects + .iter() + .filter(|e| matches!(e, Effect::RecoverComponent { .. })) + .count(), + 1, + "exactly one recovery attempt before exhaustion", + ); + assert!(effects.contains(&Effect::AssertReset(C0))); +} + +/// A `RecoveryUnavailable` for a component other than the one currently under +/// recovery is dropped — same guard `Restored` already gets. +#[test] +fn recovery_unavailable_other_component_dropped() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C0), // wrong target: dropped + ], + ); + assert_eq!(state, State::Recovering(C1)); + assert!(!effects.contains(&Effect::AssertReset(C1))); + assert!(!effects.contains(&Effect::ReportIsolated(C1))); +} + +/// A `RecoveryUnavailable` for an id outside the chain is dropped at the +/// dispatch boundary, before any handler sees it. +#[test] +fn recovery_unavailable_off_chain_dropped() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_required()), + (C1, ComponentAttrs::passive_isolable()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationFailed(C1), // → Recovering(C1) + Event::RecoveryUnavailable(C2), // C2 is not in this chain + ], + ); + assert_eq!(state, State::Recovering(C1)); + assert!(!effects.contains(&Effect::AssertReset(C1))); +} + /// A component that recovers within its retry budget is not degraded, so /// nothing is reported — reports mark components taken *out of service*, not /// every transient failure. @@ -2288,7 +2433,7 @@ fn random_chain(rng: &mut SplitMix64) -> heapless::Vec<(ComponentId, ComponentAt /// Build one random event over the given id palette. Id-less events ignore it. fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { let id = ids[rng.below(ids.len() as u32) as usize]; - match rng.below(15) { + match rng.below(16) { 0 => Event::VerificationPassed(id), 1 => Event::VerificationFailed(id), 2 => Event::ComponentReady(id), @@ -2303,6 +2448,7 @@ fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { 11 => Event::UpdateRejected, 12 => Event::RecoveryFailed, 13 => Event::CommitTimeout, + 14 => Event::RecoveryUnavailable(id), _ => Event::EffectFailed, } }