diff --git a/services/orchestrator/adapters/walk/BUILD.bazel b/services/orchestrator/adapters/walk/BUILD.bazel new file mode 100644 index 00000000..cf86946e --- /dev/null +++ b/services/orchestrator/adapters/walk/BUILD.bazel @@ -0,0 +1,24 @@ +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "orchestrator_checkpoint_walk", + srcs = [ + "src/lib.rs", + "src/walk.rs", + ], + edition = "2024", + visibility = ["//visibility:public"], + deps = [ + "//services/orchestrator/capabilities:orchestrator_capabilities", + "//services/orchestrator/config:orchestrator_config", + ], +) + +# Host tests: build on the host platform, no kernel/QEMU. +rust_test( + name = "orchestrator_checkpoint_walk_test", + crate = ":orchestrator_checkpoint_walk", +) diff --git a/services/orchestrator/adapters/walk/src/lib.rs b/services/orchestrator/adapters/walk/src/lib.rs new file mode 100644 index 00000000..2eefd3c6 --- /dev/null +++ b/services/orchestrator/adapters/walk/src/lib.rs @@ -0,0 +1,17 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! Concrete [`BootWatch`] that walks a device's boot checkpoints in order, +//! polling an [`EvidenceReader`] for each one and judging the per-checkpoint +//! windows against the caller-injected `now_millis`. +//! +//! Generic over the reader (`R`) and probe vocabulary (`P`), so the same +//! walker serves GPIO-backed boards, register-backed SoCs, and test +//! doubles. Board wiring constructs one per component and hands them to +//! the platform driver as `Board::boot_watches`. + +#![cfg_attr(not(test), no_std)] + +mod walk; + +pub use walk::CheckpointWalk; diff --git a/services/orchestrator/adapters/walk/src/walk.rs b/services/orchestrator/adapters/walk/src/walk.rs new file mode 100644 index 00000000..fadd52de --- /dev/null +++ b/services/orchestrator/adapters/walk/src/walk.rs @@ -0,0 +1,525 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, FailureCause, WalkVerdict}; +use orchestrator_config::BootCheckpoint; + +/// Walks a device's [`BootCheckpoint`]s in declaration order, reading +/// evidence from an [`EvidenceReader`] at each poll. The checkpoint list +/// comes from the device table and is `&'static`: lifetimes match the +/// board config. +/// +/// Construction panics on an empty checkpoint list (a device with no +/// checkpoints is unwatchable). A read error is treated as silence +/// (`Booting`), so a transient bus glitch does not kill a healthy boot. +/// A lapsed window is a timeout with no final read: a device that has not +/// reported cannot be judged on a race. Each checkpoint's deadline +/// starts from the first poll after arm, not from the arm call, so +/// time between arming and polling does not count against the window. +/// An unarmed or post-terminal poll +/// returns `Waiting { deadline_millis: u64::MAX }` (no deadline). +pub struct CheckpointWalk { + reader: R, + checkpoints: &'static [BootCheckpoint

], + phase: Phase, +} + +enum Phase { + Idle, + Armed, + Walking { cursor: usize, deadline_millis: u64 }, +} + +impl CheckpointWalk { + /// Binds a reader and its checkpoint list into a walk. The checkpoints + /// are walked in declaration order; each one's probe is resolved by + /// the reader. + /// + /// # Panics + /// + /// Panics if `checkpoints` is empty. + pub fn new(reader: R, checkpoints: &'static [BootCheckpoint

]) -> Self { + // DeviceConfig::new already rejects empty checkpoint lists at build + // time, so this only fires if someone constructs a walk by hand. + assert!(!checkpoints.is_empty(), "checkpoint list must not be empty"); + Self { + reader, + checkpoints, + phase: Phase::Idle, + } + } + + /// Mutable access to the reader. + pub fn reader_mut(&mut self) -> &mut R { + &mut self.reader + } +} + +impl, P> BootWatch for CheckpointWalk { + fn arm(&mut self) { + self.phase = Phase::Armed; + } + + fn poll(&mut self, now_millis: u64) -> WalkVerdict { + if let Phase::Armed = self.phase { + let timeout_millis = self.checkpoints[0].timeout().as_millis() as u64; + let deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: 0, + deadline_millis: deadline, + }; + } + + let (cursor, deadline) = match self.phase { + Phase::Walking { + cursor, + deadline_millis, + } => (cursor, deadline_millis), + _ => { + return WalkVerdict::Waiting { + deadline_millis: u64::MAX, + } + } + }; + + if now_millis >= deadline { + self.phase = Phase::Idle; + return WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::TimedOut, + }; + } + + let status = self + .reader + .read(self.checkpoints[cursor].probe()) + .unwrap_or(BootStatus::Booting); + + match status { + BootStatus::Booting => WalkVerdict::Waiting { + deadline_millis: deadline, + }, + BootStatus::Booted => { + let next = cursor + 1; + if next == self.checkpoints.len() { + self.phase = Phase::Idle; + WalkVerdict::Complete + } else { + let timeout_millis = self.checkpoints[next].timeout().as_millis() as u64; + let new_deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: next, + deadline_millis: new_deadline, + }; + WalkVerdict::Waiting { + deadline_millis: new_deadline, + } + } + } + BootStatus::FailedRetriable => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceRetriable, + } + } + BootStatus::FailedFatal => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceFatal, + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::time::Duration; + + const BL1: BootCheckpoint = BootCheckpoint::new("bl1", 1, Duration::from_millis(100)); + const KERNEL: BootCheckpoint = BootCheckpoint::new("kernel", 2, Duration::from_millis(200)); + const CHECKPOINTS: &[BootCheckpoint] = &[BL1, KERNEL]; + + // A progress-register reader: probe N is Booted once progress >= N. + // Mirrors the SocReader archetype in the evidence tests. + struct ProgressReader { + level: u8, + fault: Option, + fail_read: bool, + } + + impl ProgressReader { + fn new() -> Self { + Self { + level: 0, + fault: None, + fail_read: false, + } + } + } + + #[derive(Debug)] + struct ReadFault; + + impl core::fmt::Display for ReadFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("read fault") + } + } + + impl core::error::Error for ReadFault {} + + impl EvidenceReader for ProgressReader { + type Error = ReadFault; + + fn read(&mut self, probe: &u8) -> Result { + if self.fail_read { + return Err(ReadFault); + } + if let Some(fault) = self.fault { + return Ok(fault); + } + Ok(if self.level >= *probe { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } + } + + fn walk() -> CheckpointWalk { + CheckpointWalk::new(ProgressReader::new(), CHECKPOINTS) + } + + // ── Happy path ────────────────────────────────────────────────────── + + #[test] + fn two_checkpoint_walk_completes_when_both_pass() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + }, + "bl1 passed, now waiting on kernel" + ); + + let v = w.poll(0); + assert_eq!(v, WalkVerdict::Complete); + } + + #[test] + fn single_checkpoint_walk_completes_in_one_poll() { + const ONE: &[BootCheckpoint] = &[BL1]; + let mut w = CheckpointWalk::new(ProgressReader::new(), ONE); + w.reader_mut().level = 1; + w.arm(); + + assert_eq!(w.poll(0), WalkVerdict::Complete); + } + + #[test] + fn progress_between_polls_advances_the_walk() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + w.reader_mut().level = 1; + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + }, + "bl1 passed at t=50, kernel deadline = 50 + 200" + ); + + w.reader_mut().level = 2; + let v = w.poll(100); + assert_eq!(v, WalkVerdict::Complete); + } + + // ── Timeout ───────────────────────────────────────────────────────── + + #[test] + fn first_checkpoint_times_out_when_device_is_silent() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + let v = w.poll(100); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + } + ); + } + + #[test] + fn second_checkpoint_times_out_after_first_passes() { + let mut w = walk(); + w.arm(); + + w.reader_mut().level = 1; + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + + let v = w.poll(200); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::TimedOut, + } + ); + } + + // ── Device-reported failures ──────────────────────────────────────── + + #[test] + fn retriable_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedRetriable); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceRetriable, + } + ); + } + + #[test] + fn fatal_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedFatal); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Read errors ───────────────────────────────────────────────────── + + #[test] + fn read_error_treated_as_silence() { + let mut w = walk(); + w.arm(); + w.reader_mut().fail_read = true; + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + }, + "bus glitch does not kill a healthy boot" + ); + + // Clear the fault and advance: the walk continues. + w.reader_mut().fail_read = false; + w.reader_mut().level = 2; + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 210 + } + ); + assert_eq!(w.poll(10), WalkVerdict::Complete); + } + + // ── arm() rewinds ─────────────────────────────────────────────────── + + #[test] + fn arm_rewinds_to_the_first_checkpoint() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + assert_eq!(w.poll(0), WalkVerdict::Complete); + + // Re-arm: back to checkpoint 0. + w.reader_mut().level = 0; + w.arm(); + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + }, + "fresh deadline from the re-arm" + ); + } + + #[test] + fn arm_mid_walk_restarts_from_the_beginning() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // passes bl1, now at kernel + + w.arm(); // restart + w.reader_mut().level = 0; + let v = w.poll(500); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 600 + }, + "restarted at bl1 with a fresh deadline" + ); + } + + // ── Unarmed / idle ───────────────────────────────────────────────── + + #[test] + fn unarmed_walk_waits_indefinitely() { + let mut w = walk(); + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + #[test] + fn idle_after_terminal_waits_indefinitely() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + let v = w.poll(100); // now >= deadline -> TimedOut + assert!(matches!(v, WalkVerdict::Failed { .. })); + + assert_eq!( + w.poll(200), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + // ── Deadline arithmetic ───────────────────────────────────────────── + + #[test] + fn deadline_is_relative_to_first_poll_not_arm() { + let mut w = walk(); + w.arm(); + // First poll at t=1000: deadline should be 1000 + 100, not 0 + 100. + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + } + ); + } + + #[test] + fn next_checkpoint_deadline_is_relative_to_the_passing_poll() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + + // bl1 passes at t=50, kernel deadline = 50 + 200. + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + } + ); + } + + // ── Lapsed window = timeout, no last-chance read ──────────────────── + + #[test] + fn booted_at_expiry_is_still_timeout() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + + w.reader_mut().level = 1; + let v = w.poll(100); // device ready, but window already lapsed + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + }, + "timeout checked before reading: a lapsed window is final" + ); + } + + // ── Fault at a later checkpoint names it correctly ──────────────── + + #[test] + fn device_fault_at_second_checkpoint_names_it() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // bl1 passes, now at kernel + + w.reader_mut().fault = Some(BootStatus::FailedFatal); + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Construction ──────────────────────────────────────────────────── + + #[test] + #[should_panic(expected = "checkpoint list must not be empty")] + fn empty_checkpoints_panic_at_construction() { + let empty: &'static [BootCheckpoint] = &[]; + CheckpointWalk::new(ProgressReader::new(), empty); + } +} diff --git a/services/orchestrator/driver/src/driver.rs b/services/orchestrator/driver/src/driver.rs index 0176f0b8..c012d4ee 100644 --- a/services/orchestrator/driver/src/driver.rs +++ b/services/orchestrator/driver/src/driver.rs @@ -4,12 +4,14 @@ //! The [`PlatformDriver`]: one executor method per [`Effect`] variant, routed from //! the SM through the [`Platform`] impl. -use openprot_orchestrator_sm::{ComponentId, ComponentKind, Effect, EffectError, Event, Platform}; +use openprot_orchestrator_sm::{ + BootFailureKind, ComponentId, ComponentKind, Effect, EffectError, Event, Platform, +}; use crate::board::{ Board, BoardCapabilities, ImageSource, Report, ReportSink, SvnFloorBinding, Verdict, Verifier, }; -use orchestrator_capabilities::{BootControl, BootWatch, Svn, SvnFloor, WalkVerdict}; +use orchestrator_capabilities::{BootControl, BootWatch, FailureCause, Svn, SvnFloor, WalkVerdict}; /// Why the driver could not carry out an effect. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -159,7 +161,7 @@ impl PlatformDriver { /// Release `id` from reset and arm its boot walk; /// [`poll_boot_walks`](Self::poll_boot_walks) feeds the verdict back - /// as `ComponentReady(id)`/`Booted(id)`/`Timeout(id)`. Arms on every + /// as `ComponentReady(id)`/`Booted(id)`/`BootFailed { id, .. }`. Arms on every /// release: a retry re-release starts a fresh walk. pub fn release_reset(&mut self, id: ComponentId) -> Result<(), DriverError> { self.boot_control(id)? @@ -175,7 +177,7 @@ impl PlatformDriver { /// Hold `id` in reset — a durable quiesce, not a pulse; at-rest /// verification and the recovery re-walk depend on it. Also stops the /// boot walk: a held device produces no boot signal, so polling it - /// could only yield a stale `Timeout`. + /// could only yield a stale `BootFailed`. pub fn assert_reset(&mut self, id: ComponentId) -> Result<(), DriverError> { self.boot_control(id)? .hold_in_reset() @@ -187,8 +189,8 @@ impl PlatformDriver { /// Polls every watched walk at `now_millis` and returns the first /// terminal verdict as its event: [`WalkVerdict::Complete`] becomes /// `ComponentReady(id)` (`Active`) or `Booted(id)` (`Passive`), - /// [`WalkVerdict::Failed`] becomes `Timeout(id)` regardless of cause — - /// retry budgeting is the SM's. The finished walk stops being watched; + /// [`WalkVerdict::Failed`] becomes `BootFailed { id, checkpoint, kind }`. + /// The finished walk stops being watched; /// each verdict is delivered once. /// /// Returns at the first event; drain by calling until @@ -220,10 +222,19 @@ impl PlatformDriver { next_deadline_millis, }; } - WalkVerdict::Failed { .. } => { + WalkVerdict::Failed { checkpoint, cause } => { self.watching[idx] = false; + let kind = match cause { + FailureCause::TimedOut => BootFailureKind::TimedOut, + FailureCause::DeviceRetriable => BootFailureKind::DeviceRetriable, + FailureCause::DeviceFatal => BootFailureKind::DeviceFatal, + }; return BootWalkPoll { - event: Some(Event::Timeout(id)), + event: Some(Event::BootFailed { + id, + checkpoint, + kind, + }), next_deadline_millis, }; } diff --git a/services/orchestrator/driver/src/lib.rs b/services/orchestrator/driver/src/lib.rs index b2644add..d2f9e2df 100644 --- a/services/orchestrator/driver/src/lib.rs +++ b/services/orchestrator/driver/src/lib.rs @@ -23,7 +23,7 @@ //! //! Boot-walk verdicts are the one asynchronous read: the run loop calls //! [`PlatformDriver::poll_boot_walks`] and dispatches the returned events -//! (`ComponentReady`/`Booted`/`Timeout`) into the SM. +//! (`ComponentReady`/`Booted`/`BootFailed`) into the SM. //! //! [`Platform`]: openprot_orchestrator_sm::Platform diff --git a/services/orchestrator/driver/src/tests.rs b/services/orchestrator/driver/src/tests.rs index 1dc9f004..9f3051ae 100644 --- a/services/orchestrator/driver/src/tests.rs +++ b/services/orchestrator/driver/src/tests.rs @@ -5,8 +5,8 @@ extern crate std; use crate::*; use openprot_orchestrator_sm::{ - ComponentAttrs, ComponentId, ComponentKind, Effect, Event, Orchestrator, Platform, - PowerOnResult, State, + BootFailureKind, ComponentAttrs, ComponentId, ComponentKind, Effect, Event, Orchestrator, + Platform, PowerOnResult, State, }; use orchestrator_capabilities::{BootWatch, FailureCause, Svn, SvnFloor, WalkVerdict}; @@ -647,12 +647,10 @@ fn completed_walks_report_by_kind() { assert_eq!(quiet.next_deadline_millis, None, "no walk left waiting"); } -// A failed walk becomes Timeout(id) regardless of cause — the retry -// decision is the SM's. -// TODO: the SM only knows Timeout, so DeviceFatal still spends retry -// budget. Add a fatal, unrecoverable-error event to the SM in a later PR. +// A failed walk becomes BootFailed, preserving the checkpoint name and +// classified cause so the SM can differentiate retry decisions later. #[test] -fn failed_walks_map_to_timeout() { +fn failed_walks_map_to_boot_failed() { let mut driver = walk_driver( [ MockWalk::scripted(std::vec![WalkVerdict::Failed { @@ -669,8 +667,22 @@ fn failed_walks_map_to_timeout() { driver.release_reset(C0).unwrap(); driver.release_reset(C1).unwrap(); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C0))); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C1))); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }) + ); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C1, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }) + ); assert_eq!(driver.poll_boot_walks(0).event, None); } @@ -780,12 +792,23 @@ fn rerelease_arms_a_fresh_walk() { ); driver.release_reset(C0).unwrap(); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C0))); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }) + ); driver.release_reset(C0).unwrap(); assert_eq!( driver.poll_boot_walks(0).event, - Some(Event::Timeout(C0)), + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }), "fresh attempt from the first checkpoint, not the old walk resumed" ); } @@ -810,12 +833,11 @@ fn booted_walk_settles_in_ready() { assert_eq!(orch.state(), State::Ready); } -// End to end, failure path: the released component never reports in, its -// Timeout enters recovery, and with no recovery capability composed yet -// the machine fails closed. The Recovery PR replaces this test with the -// recovery-path one — its failure there is the reminder. +// End to end, failure path: the released component's walk fails, its +// BootFailed enters recovery, and with no recovery capability composed +// yet the machine fails closed. #[test] -fn boot_timeout_fails_closed_without_recovery() { +fn boot_failure_fails_closed_without_recovery() { let mut orch = orchestrator(); let mut driver = PlatformDriver::::new(Board { boot_watches: [MockWalk::scripted(std::vec![WalkVerdict::Failed { @@ -829,7 +851,14 @@ fn boot_timeout_fails_closed_without_recovery() { assert_eq!(orch.state(), State::Ready); let event = driver.poll_boot_walks(0).event.expect("walk failed"); - assert_eq!(event, Event::Timeout(C0)); + assert_eq!( + event, + Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + } + ); orch.dispatch(&mut driver, event); assert_eq!(orch.state(), State::Locked); diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index e049c816..bbac19b7 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs @@ -634,13 +634,11 @@ impl Rot { self.clear_awaiting_boot(*id); Outcome::Handled } - // Device-agnostic boot-progress watchdog. A passive component - // released speculatively can miss its window while the walk is - // still in `PreSupervision`; treat that as a boot failure and - // recover it, exactly as the supervised states do. A timeout for - // a component not awaiting boot (e.g. still under verification) - // is spurious and dropped. - Event::Timeout(id) => { + // Boot failure: either the walk judged a checkpoint failure + // (BootFailed) or the fleet-level watchdog fired (Timeout). + // Both recover the component if it is still awaiting boot; + // stale/spurious events are dropped. + Event::BootFailed { id, .. } | Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { @@ -865,12 +863,10 @@ impl Rot { self.clear_awaiting_boot(*id); Outcome::Handled } - // Device-agnostic boot-progress watchdog across every supervised - // state: a released component that never reported in before its - // window closed is recovered like any other boot failure. A timeout - // for a component not awaiting boot (already reported, gated, or - // never released) is stale/spurious and dropped. - Event::Timeout(id) => { + // Boot failure across every supervised state: a walk checkpoint + // failure (BootFailed) or fleet-level watchdog (Timeout) recovers + // the component if it is still awaiting boot. Stale events dropped. + Event::BootFailed { id, .. } | Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index f7458e5f..8d25d010 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs @@ -185,6 +185,20 @@ pub enum PowerOnResult { SelfVerificationFailed, } +/// Why a boot walk failed at a checkpoint, as reported by the platform +/// driver. Mirrors `orchestrator_capabilities::FailureCause` without +/// coupling the state machine crate to the capabilities crate. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum BootFailureKind { + /// The checkpoint's window expired; the device reported nothing. + TimedOut, + /// The device reported a failure worth another attempt. + DeviceRetriable, + /// The device reported a terminal failure; re-running the same image + /// cannot change the verdict. + DeviceFatal, +} + /// Everything the outside world can tell the state machine. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Event { @@ -226,10 +240,20 @@ pub enum Event { RecoveryUnavailable(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, + /// A boot walk failed at a specific checkpoint with a classified cause. + /// Produced by the platform driver when `CheckpointWalk::poll` returns + /// `WalkVerdict::Failed`. A component awaiting boot enters recovery. + /// The `checkpoint` and `kind` fields are preserved so the state + /// machine can differentiate (e.g. skip retries on `DeviceFatal`). + BootFailed { + id: ComponentId, + checkpoint: &'static str, + kind: BootFailureKind, + }, /// The platform driver's boot-progress watchdog fired: `id` did not report its /// boot-progress signal ([`Event::ComponentReady`] for an `Active` /// component, [`Event::Booted`] for a `Passive` one) within its configured - /// boot timeout. Treated as a verification failure — a component still + /// boot timeout. Treated as a verification failure, a component still /// awaiting boot-progress enters recovery. A timeout for a component that is /// not awaiting boot (never released, already reported in, or already gated) /// is stale/spurious and dropped. The watchdog is per component and @@ -273,6 +297,7 @@ impl Event { | Event::CorruptionDetected(id) | Event::Restored(id) | Event::RecoveryUnavailable(id) + | Event::BootFailed { id, .. } | Event::Timeout(id) => Some(*id), Event::PowerGood(_) | Event::AttestationChallenge diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index 21425d17..080a1e74 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs @@ -567,6 +567,52 @@ fn timeout_stale_id_ignored() { assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); } +/// A `BootFailed` for the awaited component enters recovery, same as +/// `Timeout`. The checkpoint and kind are preserved in the event but do not +/// affect the transition. +#[test] +fn boot_failed_awaited_enters_recovering() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::DeviceRetriable, + }, + ], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); +} + +/// A `BootFailed` for a component not awaiting boot is stale and dropped. +#[test] +fn boot_failed_stale_id_ignored() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C1, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }, + ], + ); + assert_eq!(state, State::AwaitingReady(Some(C0))); + assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); +} + /// An out-of-chain id in a `VerificationFailed` report is dropped: the core /// supervises only chain components, so a verdict for an id the chain does not /// contain neither enters `Recovering` nor emits `RecoverComponent`. @@ -2441,7 +2487,24 @@ fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { 4 => Event::BootConfirmed(id), 5 => Event::CorruptionDetected(id), 6 => Event::Restored(id), - 7 => Event::Timeout(id), + 7 => { + // Coin-flip: exercise both BootFailed and Timeout on + // this arm. + if rng.below(2) == 0 { + let kind = match rng.below(3) { + 0 => BootFailureKind::TimedOut, + 1 => BootFailureKind::DeviceRetriable, + _ => BootFailureKind::DeviceFatal, + }; + Event::BootFailed { + id, + checkpoint: "fuzz", + kind, + } + } else { + Event::Timeout(id) + } + } 8 => Event::AttestationChallenge, 9 => Event::UpdateRequest, 10 => Event::UpdateVerified, diff --git a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel index d8d6a98f..e80197d9 100644 --- a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel +++ b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel @@ -61,6 +61,8 @@ rust_app( tags = ["kernel"], target_compatible_with = TARGET_COMPATIBLE_WITH, deps = [ + "//services/orchestrator/adapters/walk:orchestrator_checkpoint_walk", + "//services/orchestrator/capabilities:orchestrator_capabilities", "//services/orchestrator/config:orchestrator_config", "//services/orchestrator/server:orchestrator_server", "//services/orchestrator/sm:orchestrator_sm", diff --git a/target/ast10x0/tests/orchestrator/runtime/main.rs b/target/ast10x0/tests/orchestrator/runtime/main.rs index ce2f3ad5..ac612bf4 100644 --- a/target/ast10x0/tests/orchestrator/runtime/main.rs +++ b/target/ast10x0/tests/orchestrator/runtime/main.rs @@ -1,28 +1,20 @@ // Licensed under the Apache-2.0 license // SPDX-License-Identifier: Apache-2.0 -//! Orchestrator integration QEMU test: all four subcomponents wired end to end -//! under the kernel — the pure core ([`Orchestrator`], `orchestrator-sm`), the -//! server runtime ([`BootWatchdogs`], `orchestrator-server`) which wraps the -//! watchdog keeper (`orchestrator-timer`), and the board device table -//! ([`DeviceConfig`], `orchestrator-config`). +//! Orchestrator integration QEMU test: the pure core (`Orchestrator`, +//! `orchestrator-sm`), the server runtime (`BootWatchdogs`, +//! `orchestrator-server`), the device table (`DeviceConfig`, +//! `orchestrator-config`), and the checkpoint walker (`CheckpointWalk`, +//! `orchestrator-checkpoint-walk`) wired end to end under the kernel. //! -//! The runtime owns the clock and the mapping, so the platform driver stays -//! thin: -//! - boot windows come from the device table ([`BootCheckpoint::timeout`]); -//! the platform driver only converts `core::time::Duration` to the -//! kernel's [`Duration`] at the arm site. -//! - [`BootWatchdogs::arm_boot`] takes that *relative* window; the runtime -//! computes the absolute deadline. -//! - [`BootWatchdogs::wait_deadline`] is handed straight to `object_wait`. -//! - [`BootWatchdogs::poll_expired`] yields the `Event`s the core consumes — -//! no mapping in the platform driver. +//! Scenarios 1-2 exercise `CheckpointWalk` end to end: the walk judges +//! per-checkpoint windows via `poll(now_millis)` and an `EvidenceReader`, +//! the runtime converts `deadline_millis` to kernel ticks via `object_wait`, +//! and a `DeadlineExceeded` re-poll proves the timeout path. //! -//! Coverage: the *inner checkpoint walk* (`bl1` → `kernel`, re-armed through the -//! runtime) for a single component, the *outer component walk* across a -//! multi-component chain (nearest-of-many deadlines, correct-id recovery), and -//! the commit watchdog. The interrupt object (IRQ 44, self-fired) stands in for -//! a component reaching a checkpoint. +//! Scenarios 3-4 exercise `BootWatchdogs` multiplexing across a +//! multi-component chain (nearest-of-many deadlines, correct-id recovery). +//! Scenario 5 covers the commit watchdog. #![no_main] #![no_std] @@ -30,12 +22,14 @@ use app_test_runtime::{constants, handle, signals}; use openprot_orchestrator_server::BootWatchdogs; use openprot_orchestrator_sm::{ - Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, Platform, - PowerOnResult, State, + BootFailureKind, Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, + Platform, PowerOnResult, State, }; +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, FailureCause, WalkVerdict}; +use orchestrator_checkpoint_walk::CheckpointWalk; use orchestrator_config::{BootCheckpoint, DeviceConfig}; use pw_status::{Error, Result}; -use userspace::time::Duration; +use userspace::time::{Clock, Duration, Instant, SystemClock}; use userspace::{entry, syscall}; /// The components this test supervises. @@ -56,13 +50,14 @@ type Watchdogs = BootWatchdogs; /// The device table: per-checkpoint windows, exactly as a board would declare /// them. Two checkpoints so the inner walk exercises re-arm-on-progress -/// (`bl1` then `kernel`). +/// (`bl1` then `kernel`). Probe ids are progress thresholds: the reader +/// reports `Booted` once its internal level reaches the threshold. const SOC: DeviceConfig = DeviceConfig::new( "soc", 0, &[ - BootCheckpoint::new("bl1", 0, core::time::Duration::from_millis(50)), - BootCheckpoint::new("kernel", 0, core::time::Duration::from_millis(50)), + BootCheckpoint::new("bl1", 1, core::time::Duration::from_millis(500)), + BootCheckpoint::new("kernel", 2, core::time::Duration::from_millis(500)), ], ); @@ -72,6 +67,40 @@ fn window(timeout: core::time::Duration) -> Duration { Duration::from_millis(timeout.as_millis() as u64) } +/// Progress-register reader for the walk: probe N is `Booted` once +/// `level >= N`. Mirrors the SocReader archetype in the evidence tests. +struct ProgressReader { + level: u8, +} + +impl EvidenceReader for ProgressReader { + type Error = core::convert::Infallible; + + fn read(&mut self, probe: &u8) -> core::result::Result { + Ok(if self.level >= *probe { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } +} + +fn ticks_to_millis(ticks: u64) -> u64 { + ticks * 1000 / SystemClock::TICKS_PER_SEC +} + +fn millis_to_ticks(millis: u64) -> u64 { + millis * SystemClock::TICKS_PER_SEC / 1000 +} + +fn failure_kind(cause: FailureCause) -> BootFailureKind { + match cause { + FailureCause::TimedOut => BootFailureKind::TimedOut, + FailureCause::DeviceRetriable => BootFailureKind::DeviceRetriable, + FailureCause::DeviceFatal => BootFailureKind::DeviceFatal, + } +} + /// A fake [`Platform`] for the run loop. It records the `ReleaseReset(id)` /// effects that open each component's boot supervision; every other effect is /// accepted so the core can settle. @@ -125,47 +154,55 @@ fn drive_releases(core: &mut Core, plat: &mut FakePlatform, ids: &[ComponentId]) Ok(()) } -/// Run one component's inner checkpoint walk through the runtime and return its -/// single terminal event. `reached` simulates the device: it fires its progress -/// signal for the first `reached` checkpoints, then goes quiet — so -/// `reached == len` boots, anything less times out at checkpoint `reached`. -fn checkpoint_walk( - wd: &mut Watchdogs, +/// Run one component's inner checkpoint walk through `CheckpointWalk` and +/// return its terminal verdict as an event. `reached` simulates the device: +/// the reader's progress advances for the first `reached` checkpoints, then +/// goes quiet, so `reached == len` boots and anything less times out. +/// +/// The walk judges per-checkpoint windows; the runtime (`object_wait`) just +/// sleeps until the walk's deadline or a device signal. +fn walk_device( + walk: &mut CheckpointWalk, id: ComponentId, - checkpoints: &[BootCheckpoint], reached: usize, ) -> Result { + walk.arm(); let mut k = 0usize; - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; - loop { - // Simulated device reaching checkpoint `k`: latch its progress signal - // before the wait (interrupt objects hold it pending, so no race). - if k < reached { - syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; - } - let deadline = wd.wait_deadline(); - match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { - Ok(wait) => { - if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { - return Err(Error::Internal); + loop { + let now = ticks_to_millis(SystemClock::now().ticks()); + match walk.poll(now) { + WalkVerdict::Waiting { deadline_millis } => { + // Simulate device progress after the poll returned Waiting, + // so every trigger pairs with a wait+ack below. + if k < reached { + walk.reader_mut().level = (k + 1) as u8; + syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; } - syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; - k += 1; - if k == checkpoints.len() { - wd.cancel_boot(id); - return Ok(Event::Booted(id)); + + let deadline = Instant::from_ticks(millis_to_ticks(deadline_millis)); + match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { + Ok(wait) => { + if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { + return Err(Error::Internal); + } + syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; + k += 1; + } + Err(Error::DeadlineExceeded) => { + // Deadline lapsed; re-poll and the walk will judge timeout. + } + Err(e) => return Err(e), } - // Forward progress: re-arm the next checkpoint through the runtime. - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; } - Err(Error::DeadlineExceeded) => { - // The window lapsed: the runtime already mapped it to an `Event`. - return wd.poll_expired().ok_or(Error::Internal); + WalkVerdict::Complete => return Ok(Event::Booted(id)), + WalkVerdict::Failed { checkpoint, cause } => { + return Ok(Event::BootFailed { + id, + checkpoint, + kind: failure_kind(cause), + }); } - Err(e) => return Err(e), } } } @@ -192,14 +229,12 @@ fn confirm(wd: &mut Watchdogs, id: ComponentId) -> Result<()> { } /// Inner walk, happy path: a single component passes every checkpoint (windows -/// from the device table, re-armed through the runtime), the walk yields -/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op — the -/// watchdog was retired by the confirmation. +/// from the device table, judged by `CheckpointWalk`), the walk yields +/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op. fn scenario_checkpoint_confirmed() -> Result<()> { pw_log::info!("scenario 1: checkpoint walk confirmed"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; if core.state() != State::Ready { @@ -207,18 +242,20 @@ fn scenario_checkpoint_confirmed() -> Result<()> { return Err(Error::Internal); } - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), SOC.checkpoints().len())?; + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, SOC.checkpoints().len())?; if terminal != Event::Booted(C0) { pw_log::error!("scenario 1: walk did not confirm boot"); return Err(Error::Internal); } + core.dispatch(&mut plat, terminal); if core.state() != State::Ready { pw_log::error!("scenario 1: core left Ready after boot confirmed"); return Err(Error::Internal); } - // The watchdog is retired: a stale timeout must not re-open recovery. + // A stale timeout must not re-open recovery. core.dispatch(&mut plat, Event::Timeout(C0)); if core.state() != State::Ready { pw_log::error!("scenario 1: stale timeout re-opened recovery"); @@ -230,20 +267,27 @@ fn scenario_checkpoint_confirmed() -> Result<()> { } /// Inner walk, timeout path: the device never signals, the first checkpoint's -/// window lapses, the runtime surfaces `Timeout`, and the core recovers. +/// window lapses (judged by `CheckpointWalk`), and the walk surfaces +/// `BootFailed` with `TimedOut`. The core recovers from the walk's verdict. fn scenario_checkpoint_timeout() -> Result<()> { pw_log::info!("scenario 2: checkpoint walk timeout drives recovery"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), 0)?; - if terminal != Event::Timeout(C0) { - pw_log::error!("scenario 2: walk did not time out"); + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, 0)?; + let is_boot_failed = matches!( + terminal, + Event::BootFailed { id, checkpoint: "bl1", kind, .. } + if id == C0 && kind == BootFailureKind::TimedOut + ); + if !is_boot_failed { + pw_log::error!("scenario 2: walk did not produce BootFailed"); return Err(Error::Internal); } + core.dispatch(&mut plat, terminal); if core.state() != State::Recovering(C0) { pw_log::error!("scenario 2: core did not enter recovery");