From d8337f3e7fffba3a1b287324d8f5ae853fd9cff9 Mon Sep 17 00:00:00 2001 From: Christina Quast Date: Thu, 17 Sep 2026 19:39:09 +0200 Subject: [PATCH 1/3] orchestrator: Add CheckpointWalk, the concrete BootWatch CheckpointWalk walks a device's BootCheckpoints in declaration order, polling an EvidenceReader at each step and judging per-checkpoint windows against caller-injected now_millis. Read errors are treated as silence (Booting), lapsed windows are timeouts with no last-chance read. The QEMU runtime itest (scenarios 1-2) now uses CheckpointWalk directly instead of the hand-rolled checkpoint loop, separating walk judgment from BootWatchdogs scheduling. Scenarios 3-5 (BootWatchdogs multiplexing and commit watchdog) are unchanged. Assisted-by: Claude --- .../orchestrator/adapters/walk/BUILD.bazel | 24 + .../orchestrator/adapters/walk/src/lib.rs | 17 + .../orchestrator/adapters/walk/src/walk.rs | 522 ++++++++++++++++++ .../tests/orchestrator/runtime/BUILD.bazel | 2 + .../tests/orchestrator/runtime/main.rs | 156 +++--- 5 files changed, 655 insertions(+), 66 deletions(-) create mode 100644 services/orchestrator/adapters/walk/BUILD.bazel create mode 100644 services/orchestrator/adapters/walk/src/lib.rs create mode 100644 services/orchestrator/adapters/walk/src/walk.rs diff --git a/services/orchestrator/adapters/walk/BUILD.bazel b/services/orchestrator/adapters/walk/BUILD.bazel new file mode 100644 index 000000000..cf86946ef --- /dev/null +++ b/services/orchestrator/adapters/walk/BUILD.bazel @@ -0,0 +1,24 @@ +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "orchestrator_checkpoint_walk", + srcs = [ + "src/lib.rs", + "src/walk.rs", + ], + edition = "2024", + visibility = ["//visibility:public"], + deps = [ + "//services/orchestrator/capabilities:orchestrator_capabilities", + "//services/orchestrator/config:orchestrator_config", + ], +) + +# Host tests: build on the host platform, no kernel/QEMU. +rust_test( + name = "orchestrator_checkpoint_walk_test", + crate = ":orchestrator_checkpoint_walk", +) diff --git a/services/orchestrator/adapters/walk/src/lib.rs b/services/orchestrator/adapters/walk/src/lib.rs new file mode 100644 index 000000000..2eefd3c69 --- /dev/null +++ b/services/orchestrator/adapters/walk/src/lib.rs @@ -0,0 +1,17 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! Concrete [`BootWatch`] that walks a device's boot checkpoints in order, +//! polling an [`EvidenceReader`] for each one and judging the per-checkpoint +//! windows against the caller-injected `now_millis`. +//! +//! Generic over the reader (`R`) and probe vocabulary (`P`), so the same +//! walker serves GPIO-backed boards, register-backed SoCs, and test +//! doubles. Board wiring constructs one per component and hands them to +//! the platform driver as `Board::boot_watches`. + +#![cfg_attr(not(test), no_std)] + +mod walk; + +pub use walk::CheckpointWalk; diff --git a/services/orchestrator/adapters/walk/src/walk.rs b/services/orchestrator/adapters/walk/src/walk.rs new file mode 100644 index 000000000..72305436a --- /dev/null +++ b/services/orchestrator/adapters/walk/src/walk.rs @@ -0,0 +1,522 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, FailureCause, WalkVerdict}; +use orchestrator_config::BootCheckpoint; + +/// Walks a device's [`BootCheckpoint`]s in declaration order, reading +/// evidence from an [`EvidenceReader`] at each poll. The checkpoint list +/// comes from the device table and is `&'static`: lifetimes match the +/// board config. +/// +/// Construction panics on an empty checkpoint list (a device with no +/// checkpoints is unwatchable). A read error is treated as silence +/// (`Booting`), so a transient bus glitch does not kill a healthy boot. +/// A lapsed window is a timeout with no final read: a device that has not +/// reported cannot be judged on a race. An unarmed or post-terminal poll +/// returns `Waiting { deadline_millis: u64::MAX }` (no deadline). +pub struct CheckpointWalk { + reader: R, + checkpoints: &'static [BootCheckpoint

], + phase: Phase, +} + +enum Phase { + Idle, + Armed, + Walking { cursor: usize, deadline_millis: u64 }, +} + +impl CheckpointWalk { + /// Binds a reader and its checkpoint list into a walk. The checkpoints + /// are walked in declaration order; each one's probe is resolved by + /// the reader. + /// + /// # Panics + /// + /// Panics if `checkpoints` is empty. + pub fn new(reader: R, checkpoints: &'static [BootCheckpoint

]) -> Self { + // DeviceConfig::new already rejects empty checkpoint lists at build + // time, so this only fires if someone constructs a walk by hand. + assert!(!checkpoints.is_empty(), "checkpoint list must not be empty"); + Self { + reader, + checkpoints, + phase: Phase::Idle, + } + } + + /// Mutable access to the reader. + pub fn reader_mut(&mut self) -> &mut R { + &mut self.reader + } +} + +impl, P> BootWatch for CheckpointWalk { + fn arm(&mut self) { + self.phase = Phase::Armed; + } + + fn poll(&mut self, now_millis: u64) -> WalkVerdict { + if let Phase::Armed = self.phase { + let timeout_millis = self.checkpoints[0].timeout().as_millis() as u64; + let deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: 0, + deadline_millis: deadline, + }; + } + + let (cursor, deadline) = match self.phase { + Phase::Walking { + cursor, + deadline_millis, + } => (cursor, deadline_millis), + _ => { + return WalkVerdict::Waiting { + deadline_millis: u64::MAX, + } + } + }; + + if now_millis >= deadline { + self.phase = Phase::Idle; + return WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::TimedOut, + }; + } + + let status = self + .reader + .read(self.checkpoints[cursor].probe()) + .unwrap_or(BootStatus::Booting); + + match status { + BootStatus::Booting => WalkVerdict::Waiting { + deadline_millis: deadline, + }, + BootStatus::Booted => { + let next = cursor + 1; + if next == self.checkpoints.len() { + self.phase = Phase::Idle; + WalkVerdict::Complete + } else { + let timeout_millis = self.checkpoints[next].timeout().as_millis() as u64; + let new_deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: next, + deadline_millis: new_deadline, + }; + WalkVerdict::Waiting { + deadline_millis: new_deadline, + } + } + } + BootStatus::FailedRetriable => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceRetriable, + } + } + BootStatus::FailedFatal => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceFatal, + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::time::Duration; + + const BL1: BootCheckpoint = BootCheckpoint::new("bl1", 1, Duration::from_millis(100)); + const KERNEL: BootCheckpoint = BootCheckpoint::new("kernel", 2, Duration::from_millis(200)); + const CHECKPOINTS: &[BootCheckpoint] = &[BL1, KERNEL]; + + // A progress-register reader: probe N is Booted once progress >= N. + // Mirrors the SocReader archetype in the evidence tests. + struct ProgressReader { + level: u8, + fault: Option, + fail_read: bool, + } + + impl ProgressReader { + fn new() -> Self { + Self { + level: 0, + fault: None, + fail_read: false, + } + } + } + + #[derive(Debug)] + struct ReadFault; + + impl core::fmt::Display for ReadFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("read fault") + } + } + + impl core::error::Error for ReadFault {} + + impl EvidenceReader for ProgressReader { + type Error = ReadFault; + + fn read(&mut self, probe: &u8) -> Result { + if self.fail_read { + return Err(ReadFault); + } + if let Some(fault) = self.fault { + return Ok(fault); + } + Ok(if self.level >= *probe { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } + } + + fn walk() -> CheckpointWalk { + CheckpointWalk::new(ProgressReader::new(), CHECKPOINTS) + } + + // ── Happy path ────────────────────────────────────────────────────── + + #[test] + fn two_checkpoint_walk_completes_when_both_pass() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + }, + "bl1 passed, now waiting on kernel" + ); + + let v = w.poll(0); + assert_eq!(v, WalkVerdict::Complete); + } + + #[test] + fn single_checkpoint_walk_completes_in_one_poll() { + const ONE: &[BootCheckpoint] = &[BL1]; + let mut w = CheckpointWalk::new(ProgressReader::new(), ONE); + w.reader_mut().level = 1; + w.arm(); + + assert_eq!(w.poll(0), WalkVerdict::Complete); + } + + #[test] + fn progress_between_polls_advances_the_walk() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + w.reader_mut().level = 1; + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + }, + "bl1 passed at t=50, kernel deadline = 50 + 200" + ); + + w.reader_mut().level = 2; + let v = w.poll(100); + assert_eq!(v, WalkVerdict::Complete); + } + + // ── Timeout ───────────────────────────────────────────────────────── + + #[test] + fn first_checkpoint_times_out_when_device_is_silent() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + let v = w.poll(100); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + } + ); + } + + #[test] + fn second_checkpoint_times_out_after_first_passes() { + let mut w = walk(); + w.arm(); + + w.reader_mut().level = 1; + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + + let v = w.poll(200); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::TimedOut, + } + ); + } + + // ── Device-reported failures ──────────────────────────────────────── + + #[test] + fn retriable_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedRetriable); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceRetriable, + } + ); + } + + #[test] + fn fatal_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedFatal); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Read errors ───────────────────────────────────────────────────── + + #[test] + fn read_error_treated_as_silence() { + let mut w = walk(); + w.arm(); + w.reader_mut().fail_read = true; + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + }, + "bus glitch does not kill a healthy boot" + ); + + // Clear the fault and advance: the walk continues. + w.reader_mut().fail_read = false; + w.reader_mut().level = 2; + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 210 + } + ); + assert_eq!(w.poll(10), WalkVerdict::Complete); + } + + // ── arm() rewinds ─────────────────────────────────────────────────── + + #[test] + fn arm_rewinds_to_the_first_checkpoint() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + assert_eq!(w.poll(0), WalkVerdict::Complete); + + // Re-arm: back to checkpoint 0. + w.reader_mut().level = 0; + w.arm(); + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + }, + "fresh deadline from the re-arm" + ); + } + + #[test] + fn arm_mid_walk_restarts_from_the_beginning() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // passes bl1, now at kernel + + w.arm(); // restart + w.reader_mut().level = 0; + let v = w.poll(500); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 600 + }, + "restarted at bl1 with a fresh deadline" + ); + } + + // ── Unarmed / idle ───────────────────────────────────────────────── + + #[test] + fn unarmed_walk_waits_indefinitely() { + let mut w = walk(); + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + #[test] + fn idle_after_terminal_waits_indefinitely() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + let v = w.poll(100); // now >= deadline -> TimedOut + assert!(matches!(v, WalkVerdict::Failed { .. })); + + assert_eq!( + w.poll(200), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + // ── Deadline arithmetic ───────────────────────────────────────────── + + #[test] + fn deadline_is_relative_to_first_poll_not_arm() { + let mut w = walk(); + w.arm(); + // First poll at t=1000: deadline should be 1000 + 100, not 0 + 100. + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + } + ); + } + + #[test] + fn next_checkpoint_deadline_is_relative_to_the_passing_poll() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + + // bl1 passes at t=50, kernel deadline = 50 + 200. + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + } + ); + } + + // ── Lapsed window = timeout, no last-chance read ──────────────────── + + #[test] + fn booted_at_expiry_is_still_timeout() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + + w.reader_mut().level = 1; + let v = w.poll(100); // device ready, but window already lapsed + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + }, + "timeout checked before reading: a lapsed window is final" + ); + } + + // ── Fault at a later checkpoint names it correctly ──────────────── + + #[test] + fn device_fault_at_second_checkpoint_names_it() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // bl1 passes, now at kernel + + w.reader_mut().fault = Some(BootStatus::FailedFatal); + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Construction ──────────────────────────────────────────────────── + + #[test] + #[should_panic(expected = "checkpoint list must not be empty")] + fn empty_checkpoints_panic_at_construction() { + let empty: &'static [BootCheckpoint] = &[]; + CheckpointWalk::new(ProgressReader::new(), empty); + } +} diff --git a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel index d8d6a98f5..e80197d9d 100644 --- a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel +++ b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel @@ -61,6 +61,8 @@ rust_app( tags = ["kernel"], target_compatible_with = TARGET_COMPATIBLE_WITH, deps = [ + "//services/orchestrator/adapters/walk:orchestrator_checkpoint_walk", + "//services/orchestrator/capabilities:orchestrator_capabilities", "//services/orchestrator/config:orchestrator_config", "//services/orchestrator/server:orchestrator_server", "//services/orchestrator/sm:orchestrator_sm", diff --git a/target/ast10x0/tests/orchestrator/runtime/main.rs b/target/ast10x0/tests/orchestrator/runtime/main.rs index ce2f3ad58..6fbce6e59 100644 --- a/target/ast10x0/tests/orchestrator/runtime/main.rs +++ b/target/ast10x0/tests/orchestrator/runtime/main.rs @@ -1,28 +1,20 @@ // Licensed under the Apache-2.0 license // SPDX-License-Identifier: Apache-2.0 -//! Orchestrator integration QEMU test: all four subcomponents wired end to end -//! under the kernel — the pure core ([`Orchestrator`], `orchestrator-sm`), the -//! server runtime ([`BootWatchdogs`], `orchestrator-server`) which wraps the -//! watchdog keeper (`orchestrator-timer`), and the board device table -//! ([`DeviceConfig`], `orchestrator-config`). +//! Orchestrator integration QEMU test: the pure core (`Orchestrator`, +//! `orchestrator-sm`), the server runtime (`BootWatchdogs`, +//! `orchestrator-server`), the device table (`DeviceConfig`, +//! `orchestrator-config`), and the checkpoint walker (`CheckpointWalk`, +//! `orchestrator-checkpoint-walk`) wired end to end under the kernel. //! -//! The runtime owns the clock and the mapping, so the platform driver stays -//! thin: -//! - boot windows come from the device table ([`BootCheckpoint::timeout`]); -//! the platform driver only converts `core::time::Duration` to the -//! kernel's [`Duration`] at the arm site. -//! - [`BootWatchdogs::arm_boot`] takes that *relative* window; the runtime -//! computes the absolute deadline. -//! - [`BootWatchdogs::wait_deadline`] is handed straight to `object_wait`. -//! - [`BootWatchdogs::poll_expired`] yields the `Event`s the core consumes — -//! no mapping in the platform driver. +//! Scenarios 1-2 exercise `CheckpointWalk`: the walk judges per-checkpoint +//! windows via `poll(now_millis)` and an `EvidenceReader`, while the runtime +//! uses the walk's `deadline_millis` as the `object_wait` argument. The walk +//! owns the verdict; the runtime owns the clock and wake scheduling. //! -//! Coverage: the *inner checkpoint walk* (`bl1` → `kernel`, re-armed through the -//! runtime) for a single component, the *outer component walk* across a -//! multi-component chain (nearest-of-many deadlines, correct-id recovery), and -//! the commit watchdog. The interrupt object (IRQ 44, self-fired) stands in for -//! a component reaching a checkpoint. +//! Scenarios 3-4 exercise `BootWatchdogs` multiplexing across a +//! multi-component chain (nearest-of-many deadlines, correct-id recovery). +//! Scenario 5 covers the commit watchdog. #![no_main] #![no_std] @@ -33,9 +25,11 @@ use openprot_orchestrator_sm::{ Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, Platform, PowerOnResult, State, }; +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, WalkVerdict}; +use orchestrator_checkpoint_walk::CheckpointWalk; use orchestrator_config::{BootCheckpoint, DeviceConfig}; use pw_status::{Error, Result}; -use userspace::time::Duration; +use userspace::time::{Clock, Duration, Instant, SystemClock}; use userspace::{entry, syscall}; /// The components this test supervises. @@ -56,13 +50,14 @@ type Watchdogs = BootWatchdogs; /// The device table: per-checkpoint windows, exactly as a board would declare /// them. Two checkpoints so the inner walk exercises re-arm-on-progress -/// (`bl1` then `kernel`). +/// (`bl1` then `kernel`). Signal ids are progress thresholds: the reader +/// reports `Booted` once its internal level reaches the threshold. const SOC: DeviceConfig = DeviceConfig::new( "soc", 0, &[ - BootCheckpoint::new("bl1", 0, core::time::Duration::from_millis(50)), - BootCheckpoint::new("kernel", 0, core::time::Duration::from_millis(50)), + BootCheckpoint::new("bl1", 1, core::time::Duration::from_millis(500)), + BootCheckpoint::new("kernel", 2, core::time::Duration::from_millis(500)), ], ); @@ -72,6 +67,32 @@ fn window(timeout: core::time::Duration) -> Duration { Duration::from_millis(timeout.as_millis() as u64) } +/// Progress-register reader for the walk: signal N is `Booted` once +/// `level >= N`. Mirrors the SocReader archetype in the evidence tests. +struct ProgressReader { + level: u8, +} + +impl EvidenceReader for ProgressReader { + type Error = core::convert::Infallible; + + fn read(&mut self, signal: &u8) -> core::result::Result { + Ok(if self.level >= *signal { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } +} + +fn ticks_to_millis(ticks: u64) -> u64 { + ticks * 1000 / SystemClock::TICKS_PER_SEC +} + +fn millis_to_ticks(millis: u64) -> u64 { + millis * SystemClock::TICKS_PER_SEC / 1000 +} + /// A fake [`Platform`] for the run loop. It records the `ReleaseReset(id)` /// effects that open each component's boot supervision; every other effect is /// accepted so the core can settle. @@ -125,47 +146,50 @@ fn drive_releases(core: &mut Core, plat: &mut FakePlatform, ids: &[ComponentId]) Ok(()) } -/// Run one component's inner checkpoint walk through the runtime and return its -/// single terminal event. `reached` simulates the device: it fires its progress -/// signal for the first `reached` checkpoints, then goes quiet — so -/// `reached == len` boots, anything less times out at checkpoint `reached`. -fn checkpoint_walk( - wd: &mut Watchdogs, +/// Run one component's inner checkpoint walk through `CheckpointWalk` and +/// return its terminal verdict as an event. `reached` simulates the device: +/// the reader's progress advances for the first `reached` checkpoints, then +/// goes quiet, so `reached == len` boots and anything less times out. +/// +/// The walk judges per-checkpoint windows; the runtime (`object_wait`) just +/// sleeps until the walk's deadline or a device signal. No `BootWatchdogs` +/// are involved: the walk owns the verdict, the kernel clock owns the wake. +fn walk_device( + walk: &mut CheckpointWalk, id: ComponentId, - checkpoints: &[BootCheckpoint], reached: usize, ) -> Result { + walk.arm(); let mut k = 0usize; - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; - loop { - // Simulated device reaching checkpoint `k`: latch its progress signal - // before the wait (interrupt objects hold it pending, so no race). - if k < reached { - syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; - } - let deadline = wd.wait_deadline(); - match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { - Ok(wait) => { - if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { - return Err(Error::Internal); + loop { + let now = ticks_to_millis(SystemClock::now().ticks()); + match walk.poll(now) { + WalkVerdict::Waiting { deadline_millis } => { + // Simulate device progress after the poll returned Waiting, + // so every trigger pairs with a wait+ack below. + if k < reached { + walk.reader_mut().level = (k + 1) as u8; + syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; } - syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; - k += 1; - if k == checkpoints.len() { - wd.cancel_boot(id); - return Ok(Event::Booted(id)); + + let deadline = Instant::from_ticks(millis_to_ticks(deadline_millis)); + match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { + Ok(wait) => { + if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { + return Err(Error::Internal); + } + syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; + k += 1; + } + Err(Error::DeadlineExceeded) => { + // Deadline lapsed; re-poll and the walk will judge timeout. + } + Err(e) => return Err(e), } - // Forward progress: re-arm the next checkpoint through the runtime. - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; - } - Err(Error::DeadlineExceeded) => { - // The window lapsed: the runtime already mapped it to an `Event`. - return wd.poll_expired().ok_or(Error::Internal); } - Err(e) => return Err(e), + WalkVerdict::Complete => return Ok(Event::Booted(id)), + WalkVerdict::Failed { .. } => return Ok(Event::Timeout(id)), } } } @@ -192,14 +216,12 @@ fn confirm(wd: &mut Watchdogs, id: ComponentId) -> Result<()> { } /// Inner walk, happy path: a single component passes every checkpoint (windows -/// from the device table, re-armed through the runtime), the walk yields -/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op — the -/// watchdog was retired by the confirmation. +/// from the device table, judged by `CheckpointWalk`), the walk yields +/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op. fn scenario_checkpoint_confirmed() -> Result<()> { pw_log::info!("scenario 1: checkpoint walk confirmed"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; if core.state() != State::Ready { @@ -207,7 +229,8 @@ fn scenario_checkpoint_confirmed() -> Result<()> { return Err(Error::Internal); } - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), SOC.checkpoints().len())?; + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, SOC.checkpoints().len())?; if terminal != Event::Booted(C0) { pw_log::error!("scenario 1: walk did not confirm boot"); return Err(Error::Internal); @@ -218,7 +241,7 @@ fn scenario_checkpoint_confirmed() -> Result<()> { return Err(Error::Internal); } - // The watchdog is retired: a stale timeout must not re-open recovery. + // A stale timeout must not re-open recovery. core.dispatch(&mut plat, Event::Timeout(C0)); if core.state() != State::Ready { pw_log::error!("scenario 1: stale timeout re-opened recovery"); @@ -230,16 +253,17 @@ fn scenario_checkpoint_confirmed() -> Result<()> { } /// Inner walk, timeout path: the device never signals, the first checkpoint's -/// window lapses, the runtime surfaces `Timeout`, and the core recovers. +/// window lapses (judged by `CheckpointWalk`, not by `BootWatchdogs`), the +/// walk surfaces `Timeout`, and the core recovers. fn scenario_checkpoint_timeout() -> Result<()> { pw_log::info!("scenario 2: checkpoint walk timeout drives recovery"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), 0)?; + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, 0)?; if terminal != Event::Timeout(C0) { pw_log::error!("scenario 2: walk did not time out"); return Err(Error::Internal); From f4e02dedc7504f59cf6490b3449adad05614cfcf Mon Sep 17 00:00:00 2001 From: Christina Quast Date: Mon, 21 Sep 2026 15:25:44 +0200 Subject: [PATCH 2/3] orchestrator: review fixes, document u64::MAX sentinel, clean up style Add a one-line doc on CheckpointWalk documenting that an unarmed or post-terminal poll returns Waiting { deadline_millis: u64::MAX }. Replace Box::leak with a const in single_checkpoint_walk_completes_in_one_poll, collapse a needless rebind, drop the "Decision 3:" section header, and improve an assertion message. In the QEMU test, update the module doc to name the actual falsifiable claim (DeadlineExceeded re-poll proves the timeout path) and trim doc comments that were unnecessarily opinionated about the walk/watchdog wiring decision. Assisted-by: Claude --- services/orchestrator/adapters/walk/src/walk.rs | 5 ++++- .../ast10x0/tests/orchestrator/runtime/main.rs | 17 +++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/services/orchestrator/adapters/walk/src/walk.rs b/services/orchestrator/adapters/walk/src/walk.rs index 72305436a..fadd52de0 100644 --- a/services/orchestrator/adapters/walk/src/walk.rs +++ b/services/orchestrator/adapters/walk/src/walk.rs @@ -13,7 +13,10 @@ use orchestrator_config::BootCheckpoint; /// checkpoints is unwatchable). A read error is treated as silence /// (`Booting`), so a transient bus glitch does not kill a healthy boot. /// A lapsed window is a timeout with no final read: a device that has not -/// reported cannot be judged on a race. An unarmed or post-terminal poll +/// reported cannot be judged on a race. Each checkpoint's deadline +/// starts from the first poll after arm, not from the arm call, so +/// time between arming and polling does not count against the window. +/// An unarmed or post-terminal poll /// returns `Waiting { deadline_millis: u64::MAX }` (no deadline). pub struct CheckpointWalk { reader: R, diff --git a/target/ast10x0/tests/orchestrator/runtime/main.rs b/target/ast10x0/tests/orchestrator/runtime/main.rs index 6fbce6e59..7057835a3 100644 --- a/target/ast10x0/tests/orchestrator/runtime/main.rs +++ b/target/ast10x0/tests/orchestrator/runtime/main.rs @@ -7,10 +7,10 @@ //! `orchestrator-config`), and the checkpoint walker (`CheckpointWalk`, //! `orchestrator-checkpoint-walk`) wired end to end under the kernel. //! -//! Scenarios 1-2 exercise `CheckpointWalk`: the walk judges per-checkpoint -//! windows via `poll(now_millis)` and an `EvidenceReader`, while the runtime -//! uses the walk's `deadline_millis` as the `object_wait` argument. The walk -//! owns the verdict; the runtime owns the clock and wake scheduling. +//! Scenarios 1-2 exercise `CheckpointWalk` end to end: the walk judges +//! per-checkpoint windows via `poll(now_millis)` and an `EvidenceReader`, +//! the runtime converts `deadline_millis` to kernel ticks via `object_wait`, +//! and a `DeadlineExceeded` re-poll proves the timeout path. //! //! Scenarios 3-4 exercise `BootWatchdogs` multiplexing across a //! multi-component chain (nearest-of-many deadlines, correct-id recovery). @@ -152,8 +152,7 @@ fn drive_releases(core: &mut Core, plat: &mut FakePlatform, ids: &[ComponentId]) /// goes quiet, so `reached == len` boots and anything less times out. /// /// The walk judges per-checkpoint windows; the runtime (`object_wait`) just -/// sleeps until the walk's deadline or a device signal. No `BootWatchdogs` -/// are involved: the walk owns the verdict, the kernel clock owns the wake. +/// sleeps until the walk's deadline or a device signal. fn walk_device( walk: &mut CheckpointWalk, id: ComponentId, @@ -235,6 +234,7 @@ fn scenario_checkpoint_confirmed() -> Result<()> { pw_log::error!("scenario 1: walk did not confirm boot"); return Err(Error::Internal); } + core.dispatch(&mut plat, terminal); if core.state() != State::Ready { pw_log::error!("scenario 1: core left Ready after boot confirmed"); @@ -253,8 +253,8 @@ fn scenario_checkpoint_confirmed() -> Result<()> { } /// Inner walk, timeout path: the device never signals, the first checkpoint's -/// window lapses (judged by `CheckpointWalk`, not by `BootWatchdogs`), the -/// walk surfaces `Timeout`, and the core recovers. +/// window lapses (judged by `CheckpointWalk`), and the walk surfaces +/// `Timeout`. The core recovers from the walk's verdict. fn scenario_checkpoint_timeout() -> Result<()> { pw_log::info!("scenario 2: checkpoint walk timeout drives recovery"); let mut core = new_core(&[C0])?; @@ -268,6 +268,7 @@ fn scenario_checkpoint_timeout() -> Result<()> { pw_log::error!("scenario 2: walk did not time out"); return Err(Error::Internal); } + core.dispatch(&mut plat, terminal); if core.state() != State::Recovering(C0) { pw_log::error!("scenario 2: core did not enter recovery"); From ba31b0f476a0be3c42a3ed10083a67cf24185925 Mon Sep 17 00:00:00 2001 From: Christina Quast Date: Mon, 21 Sep 2026 15:51:37 +0200 Subject: [PATCH 3/3] orchestrator: Add Event::BootFailed, preserve walk failure cause WalkVerdict::Failed carries a checkpoint name and a FailureCause (TimedOut / DeviceRetriable / DeviceFatal). Until now the driver collapsed every failure to Event::Timeout, losing the cause. Add BootFailureKind to the SM crate (mirrors FailureCause without coupling SM to capabilities) and Event::BootFailed { id, checkpoint, kind }. The driver now maps Failed to BootFailed instead of Timeout. Both events enter recovery identically today. The checkpoint and kind fields let the SM skip retries on DeviceFatal or log the failing checkpoint without a second round-trip. Adds two deterministic SM tests (boot_failed_awaited_enters_recovering, boot_failed_stale_id_ignored) alongside the existing Timeout ones. The property test fuzzer exercises BootFailed via a coin-flip on the Timeout arm. Assisted-by: Claude --- services/orchestrator/driver/src/driver.rs | 27 +++++--- services/orchestrator/driver/src/lib.rs | 2 +- services/orchestrator/driver/src/tests.rs | 63 +++++++++++++----- services/orchestrator/sm/src/lib.rs | 22 +++---- services/orchestrator/sm/src/model.rs | 27 +++++++- services/orchestrator/sm/src/tests.rs | 65 ++++++++++++++++++- .../tests/orchestrator/runtime/main.rs | 41 ++++++++---- 7 files changed, 195 insertions(+), 52 deletions(-) diff --git a/services/orchestrator/driver/src/driver.rs b/services/orchestrator/driver/src/driver.rs index 0176f0b85..c012d4eec 100644 --- a/services/orchestrator/driver/src/driver.rs +++ b/services/orchestrator/driver/src/driver.rs @@ -4,12 +4,14 @@ //! The [`PlatformDriver`]: one executor method per [`Effect`] variant, routed from //! the SM through the [`Platform`] impl. -use openprot_orchestrator_sm::{ComponentId, ComponentKind, Effect, EffectError, Event, Platform}; +use openprot_orchestrator_sm::{ + BootFailureKind, ComponentId, ComponentKind, Effect, EffectError, Event, Platform, +}; use crate::board::{ Board, BoardCapabilities, ImageSource, Report, ReportSink, SvnFloorBinding, Verdict, Verifier, }; -use orchestrator_capabilities::{BootControl, BootWatch, Svn, SvnFloor, WalkVerdict}; +use orchestrator_capabilities::{BootControl, BootWatch, FailureCause, Svn, SvnFloor, WalkVerdict}; /// Why the driver could not carry out an effect. #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -159,7 +161,7 @@ impl PlatformDriver { /// Release `id` from reset and arm its boot walk; /// [`poll_boot_walks`](Self::poll_boot_walks) feeds the verdict back - /// as `ComponentReady(id)`/`Booted(id)`/`Timeout(id)`. Arms on every + /// as `ComponentReady(id)`/`Booted(id)`/`BootFailed { id, .. }`. Arms on every /// release: a retry re-release starts a fresh walk. pub fn release_reset(&mut self, id: ComponentId) -> Result<(), DriverError> { self.boot_control(id)? @@ -175,7 +177,7 @@ impl PlatformDriver { /// Hold `id` in reset — a durable quiesce, not a pulse; at-rest /// verification and the recovery re-walk depend on it. Also stops the /// boot walk: a held device produces no boot signal, so polling it - /// could only yield a stale `Timeout`. + /// could only yield a stale `BootFailed`. pub fn assert_reset(&mut self, id: ComponentId) -> Result<(), DriverError> { self.boot_control(id)? .hold_in_reset() @@ -187,8 +189,8 @@ impl PlatformDriver { /// Polls every watched walk at `now_millis` and returns the first /// terminal verdict as its event: [`WalkVerdict::Complete`] becomes /// `ComponentReady(id)` (`Active`) or `Booted(id)` (`Passive`), - /// [`WalkVerdict::Failed`] becomes `Timeout(id)` regardless of cause — - /// retry budgeting is the SM's. The finished walk stops being watched; + /// [`WalkVerdict::Failed`] becomes `BootFailed { id, checkpoint, kind }`. + /// The finished walk stops being watched; /// each verdict is delivered once. /// /// Returns at the first event; drain by calling until @@ -220,10 +222,19 @@ impl PlatformDriver { next_deadline_millis, }; } - WalkVerdict::Failed { .. } => { + WalkVerdict::Failed { checkpoint, cause } => { self.watching[idx] = false; + let kind = match cause { + FailureCause::TimedOut => BootFailureKind::TimedOut, + FailureCause::DeviceRetriable => BootFailureKind::DeviceRetriable, + FailureCause::DeviceFatal => BootFailureKind::DeviceFatal, + }; return BootWalkPoll { - event: Some(Event::Timeout(id)), + event: Some(Event::BootFailed { + id, + checkpoint, + kind, + }), next_deadline_millis, }; } diff --git a/services/orchestrator/driver/src/lib.rs b/services/orchestrator/driver/src/lib.rs index b2644addc..d2f9e2df3 100644 --- a/services/orchestrator/driver/src/lib.rs +++ b/services/orchestrator/driver/src/lib.rs @@ -23,7 +23,7 @@ //! //! Boot-walk verdicts are the one asynchronous read: the run loop calls //! [`PlatformDriver::poll_boot_walks`] and dispatches the returned events -//! (`ComponentReady`/`Booted`/`Timeout`) into the SM. +//! (`ComponentReady`/`Booted`/`BootFailed`) into the SM. //! //! [`Platform`]: openprot_orchestrator_sm::Platform diff --git a/services/orchestrator/driver/src/tests.rs b/services/orchestrator/driver/src/tests.rs index 1dc9f0046..9f3051aec 100644 --- a/services/orchestrator/driver/src/tests.rs +++ b/services/orchestrator/driver/src/tests.rs @@ -5,8 +5,8 @@ extern crate std; use crate::*; use openprot_orchestrator_sm::{ - ComponentAttrs, ComponentId, ComponentKind, Effect, Event, Orchestrator, Platform, - PowerOnResult, State, + BootFailureKind, ComponentAttrs, ComponentId, ComponentKind, Effect, Event, Orchestrator, + Platform, PowerOnResult, State, }; use orchestrator_capabilities::{BootWatch, FailureCause, Svn, SvnFloor, WalkVerdict}; @@ -647,12 +647,10 @@ fn completed_walks_report_by_kind() { assert_eq!(quiet.next_deadline_millis, None, "no walk left waiting"); } -// A failed walk becomes Timeout(id) regardless of cause — the retry -// decision is the SM's. -// TODO: the SM only knows Timeout, so DeviceFatal still spends retry -// budget. Add a fatal, unrecoverable-error event to the SM in a later PR. +// A failed walk becomes BootFailed, preserving the checkpoint name and +// classified cause so the SM can differentiate retry decisions later. #[test] -fn failed_walks_map_to_timeout() { +fn failed_walks_map_to_boot_failed() { let mut driver = walk_driver( [ MockWalk::scripted(std::vec![WalkVerdict::Failed { @@ -669,8 +667,22 @@ fn failed_walks_map_to_timeout() { driver.release_reset(C0).unwrap(); driver.release_reset(C1).unwrap(); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C0))); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C1))); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }) + ); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C1, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }) + ); assert_eq!(driver.poll_boot_walks(0).event, None); } @@ -780,12 +792,23 @@ fn rerelease_arms_a_fresh_walk() { ); driver.release_reset(C0).unwrap(); - assert_eq!(driver.poll_boot_walks(0).event, Some(Event::Timeout(C0))); + assert_eq!( + driver.poll_boot_walks(0).event, + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }) + ); driver.release_reset(C0).unwrap(); assert_eq!( driver.poll_boot_walks(0).event, - Some(Event::Timeout(C0)), + Some(Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + }), "fresh attempt from the first checkpoint, not the old walk resumed" ); } @@ -810,12 +833,11 @@ fn booted_walk_settles_in_ready() { assert_eq!(orch.state(), State::Ready); } -// End to end, failure path: the released component never reports in, its -// Timeout enters recovery, and with no recovery capability composed yet -// the machine fails closed. The Recovery PR replaces this test with the -// recovery-path one — its failure there is the reminder. +// End to end, failure path: the released component's walk fails, its +// BootFailed enters recovery, and with no recovery capability composed +// yet the machine fails closed. #[test] -fn boot_timeout_fails_closed_without_recovery() { +fn boot_failure_fails_closed_without_recovery() { let mut orch = orchestrator(); let mut driver = PlatformDriver::::new(Board { boot_watches: [MockWalk::scripted(std::vec![WalkVerdict::Failed { @@ -829,7 +851,14 @@ fn boot_timeout_fails_closed_without_recovery() { assert_eq!(orch.state(), State::Ready); let event = driver.poll_boot_walks(0).event.expect("walk failed"); - assert_eq!(event, Event::Timeout(C0)); + assert_eq!( + event, + Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::TimedOut, + } + ); orch.dispatch(&mut driver, event); assert_eq!(orch.state(), State::Locked); diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index e049c8164..bbac19b78 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs @@ -634,13 +634,11 @@ impl Rot { self.clear_awaiting_boot(*id); Outcome::Handled } - // Device-agnostic boot-progress watchdog. A passive component - // released speculatively can miss its window while the walk is - // still in `PreSupervision`; treat that as a boot failure and - // recover it, exactly as the supervised states do. A timeout for - // a component not awaiting boot (e.g. still under verification) - // is spurious and dropped. - Event::Timeout(id) => { + // Boot failure: either the walk judged a checkpoint failure + // (BootFailed) or the fleet-level watchdog fired (Timeout). + // Both recover the component if it is still awaiting boot; + // stale/spurious events are dropped. + Event::BootFailed { id, .. } | Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { @@ -865,12 +863,10 @@ impl Rot { self.clear_awaiting_boot(*id); Outcome::Handled } - // Device-agnostic boot-progress watchdog across every supervised - // state: a released component that never reported in before its - // window closed is recovered like any other boot failure. A timeout - // for a component not awaiting boot (already reported, gated, or - // never released) is stale/spurious and dropped. - Event::Timeout(id) => { + // Boot failure across every supervised state: a walk checkpoint + // failure (BootFailed) or fleet-level watchdog (Timeout) recovers + // the component if it is still awaiting boot. Stale events dropped. + Event::BootFailed { id, .. } | Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index f7458e5fa..8d25d0103 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs @@ -185,6 +185,20 @@ pub enum PowerOnResult { SelfVerificationFailed, } +/// Why a boot walk failed at a checkpoint, as reported by the platform +/// driver. Mirrors `orchestrator_capabilities::FailureCause` without +/// coupling the state machine crate to the capabilities crate. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum BootFailureKind { + /// The checkpoint's window expired; the device reported nothing. + TimedOut, + /// The device reported a failure worth another attempt. + DeviceRetriable, + /// The device reported a terminal failure; re-running the same image + /// cannot change the verdict. + DeviceFatal, +} + /// Everything the outside world can tell the state machine. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Event { @@ -226,10 +240,20 @@ pub enum Event { RecoveryUnavailable(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, + /// A boot walk failed at a specific checkpoint with a classified cause. + /// Produced by the platform driver when `CheckpointWalk::poll` returns + /// `WalkVerdict::Failed`. A component awaiting boot enters recovery. + /// The `checkpoint` and `kind` fields are preserved so the state + /// machine can differentiate (e.g. skip retries on `DeviceFatal`). + BootFailed { + id: ComponentId, + checkpoint: &'static str, + kind: BootFailureKind, + }, /// The platform driver's boot-progress watchdog fired: `id` did not report its /// boot-progress signal ([`Event::ComponentReady`] for an `Active` /// component, [`Event::Booted`] for a `Passive` one) within its configured - /// boot timeout. Treated as a verification failure — a component still + /// boot timeout. Treated as a verification failure, a component still /// awaiting boot-progress enters recovery. A timeout for a component that is /// not awaiting boot (never released, already reported in, or already gated) /// is stale/spurious and dropped. The watchdog is per component and @@ -273,6 +297,7 @@ impl Event { | Event::CorruptionDetected(id) | Event::Restored(id) | Event::RecoveryUnavailable(id) + | Event::BootFailed { id, .. } | Event::Timeout(id) => Some(*id), Event::PowerGood(_) | Event::AttestationChallenge diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index 21425d171..080a1e74f 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs @@ -567,6 +567,52 @@ fn timeout_stale_id_ignored() { assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); } +/// A `BootFailed` for the awaited component enters recovery, same as +/// `Timeout`. The checkpoint and kind are preserved in the event but do not +/// affect the transition. +#[test] +fn boot_failed_awaited_enters_recovering() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::DeviceRetriable, + }, + ], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); +} + +/// A `BootFailed` for a component not awaiting boot is stale and dropped. +#[test] +fn boot_failed_stale_id_ignored() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C1, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }, + ], + ); + assert_eq!(state, State::AwaitingReady(Some(C0))); + assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); +} + /// An out-of-chain id in a `VerificationFailed` report is dropped: the core /// supervises only chain components, so a verdict for an id the chain does not /// contain neither enters `Recovering` nor emits `RecoverComponent`. @@ -2441,7 +2487,24 @@ fn random_event(rng: &mut SplitMix64, ids: &[ComponentId]) -> Event { 4 => Event::BootConfirmed(id), 5 => Event::CorruptionDetected(id), 6 => Event::Restored(id), - 7 => Event::Timeout(id), + 7 => { + // Coin-flip: exercise both BootFailed and Timeout on + // this arm. + if rng.below(2) == 0 { + let kind = match rng.below(3) { + 0 => BootFailureKind::TimedOut, + 1 => BootFailureKind::DeviceRetriable, + _ => BootFailureKind::DeviceFatal, + }; + Event::BootFailed { + id, + checkpoint: "fuzz", + kind, + } + } else { + Event::Timeout(id) + } + } 8 => Event::AttestationChallenge, 9 => Event::UpdateRequest, 10 => Event::UpdateVerified, diff --git a/target/ast10x0/tests/orchestrator/runtime/main.rs b/target/ast10x0/tests/orchestrator/runtime/main.rs index 7057835a3..ac612bf46 100644 --- a/target/ast10x0/tests/orchestrator/runtime/main.rs +++ b/target/ast10x0/tests/orchestrator/runtime/main.rs @@ -22,10 +22,10 @@ use app_test_runtime::{constants, handle, signals}; use openprot_orchestrator_server::BootWatchdogs; use openprot_orchestrator_sm::{ - Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, Platform, - PowerOnResult, State, + BootFailureKind, Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, + Platform, PowerOnResult, State, }; -use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, WalkVerdict}; +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, FailureCause, WalkVerdict}; use orchestrator_checkpoint_walk::CheckpointWalk; use orchestrator_config::{BootCheckpoint, DeviceConfig}; use pw_status::{Error, Result}; @@ -50,7 +50,7 @@ type Watchdogs = BootWatchdogs; /// The device table: per-checkpoint windows, exactly as a board would declare /// them. Two checkpoints so the inner walk exercises re-arm-on-progress -/// (`bl1` then `kernel`). Signal ids are progress thresholds: the reader +/// (`bl1` then `kernel`). Probe ids are progress thresholds: the reader /// reports `Booted` once its internal level reaches the threshold. const SOC: DeviceConfig = DeviceConfig::new( "soc", @@ -67,7 +67,7 @@ fn window(timeout: core::time::Duration) -> Duration { Duration::from_millis(timeout.as_millis() as u64) } -/// Progress-register reader for the walk: signal N is `Booted` once +/// Progress-register reader for the walk: probe N is `Booted` once /// `level >= N`. Mirrors the SocReader archetype in the evidence tests. struct ProgressReader { level: u8, @@ -76,8 +76,8 @@ struct ProgressReader { impl EvidenceReader for ProgressReader { type Error = core::convert::Infallible; - fn read(&mut self, signal: &u8) -> core::result::Result { - Ok(if self.level >= *signal { + fn read(&mut self, probe: &u8) -> core::result::Result { + Ok(if self.level >= *probe { BootStatus::Booted } else { BootStatus::Booting @@ -93,6 +93,14 @@ fn millis_to_ticks(millis: u64) -> u64 { millis * SystemClock::TICKS_PER_SEC / 1000 } +fn failure_kind(cause: FailureCause) -> BootFailureKind { + match cause { + FailureCause::TimedOut => BootFailureKind::TimedOut, + FailureCause::DeviceRetriable => BootFailureKind::DeviceRetriable, + FailureCause::DeviceFatal => BootFailureKind::DeviceFatal, + } +} + /// A fake [`Platform`] for the run loop. It records the `ReleaseReset(id)` /// effects that open each component's boot supervision; every other effect is /// accepted so the core can settle. @@ -188,7 +196,13 @@ fn walk_device( } } WalkVerdict::Complete => return Ok(Event::Booted(id)), - WalkVerdict::Failed { .. } => return Ok(Event::Timeout(id)), + WalkVerdict::Failed { checkpoint, cause } => { + return Ok(Event::BootFailed { + id, + checkpoint, + kind: failure_kind(cause), + }); + } } } } @@ -254,7 +268,7 @@ fn scenario_checkpoint_confirmed() -> Result<()> { /// Inner walk, timeout path: the device never signals, the first checkpoint's /// window lapses (judged by `CheckpointWalk`), and the walk surfaces -/// `Timeout`. The core recovers from the walk's verdict. +/// `BootFailed` with `TimedOut`. The core recovers from the walk's verdict. fn scenario_checkpoint_timeout() -> Result<()> { pw_log::info!("scenario 2: checkpoint walk timeout drives recovery"); let mut core = new_core(&[C0])?; @@ -264,8 +278,13 @@ fn scenario_checkpoint_timeout() -> Result<()> { let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); let terminal = walk_device(&mut walk, C0, 0)?; - if terminal != Event::Timeout(C0) { - pw_log::error!("scenario 2: walk did not time out"); + let is_boot_failed = matches!( + terminal, + Event::BootFailed { id, checkpoint: "bl1", kind, .. } + if id == C0 && kind == BootFailureKind::TimedOut + ); + if !is_boot_failed { + pw_log::error!("scenario 2: walk did not produce BootFailed"); return Err(Error::Internal); }