diff --git a/services/orchestrator/adapters/walk/BUILD.bazel b/services/orchestrator/adapters/walk/BUILD.bazel new file mode 100644 index 00000000..093bacc6 --- /dev/null +++ b/services/orchestrator/adapters/walk/BUILD.bazel @@ -0,0 +1,21 @@ +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "orchestrator_checkpoint_walk", + srcs = ["src/lib.rs"], + edition = "2024", + visibility = ["//visibility:public"], + deps = [ + "//services/orchestrator/capabilities:orchestrator_capabilities", + "//services/orchestrator/config:orchestrator_config", + ], +) + +# Host tests: build on the host platform, no kernel/QEMU. +rust_test( + name = "orchestrator_checkpoint_walk_test", + crate = ":orchestrator_checkpoint_walk", +) diff --git a/services/orchestrator/adapters/walk/src/lib.rs b/services/orchestrator/adapters/walk/src/lib.rs new file mode 100644 index 00000000..a90af76e --- /dev/null +++ b/services/orchestrator/adapters/walk/src/lib.rs @@ -0,0 +1,534 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! Concrete [`BootWatch`] that walks a device's boot checkpoints in order, +//! polling an [`EvidenceReader`] for each one and judging the per-checkpoint +//! windows against the caller-injected `now_millis`. +//! +//! Generic over the reader (`R`) and signal vocabulary (`G`), so the same +//! walker serves GPIO-backed boards, register-backed SoCs, and test +//! doubles. Board wiring constructs one per component and hands them to +//! the platform driver as `Board::boot_watches`. + +#![cfg_attr(not(test), no_std)] + +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, FailureCause, WalkVerdict}; +use orchestrator_config::BootCheckpoint; + +/// Walks a device's [`BootCheckpoint`]s in declaration order, reading +/// evidence from an [`EvidenceReader`] at each poll. The checkpoint list +/// comes from the device table and is `&'static`: lifetimes match the +/// board config. +/// +/// Construction panics on an empty checkpoint list (a device with no +/// checkpoints is unwatchable). A read error is treated as silence +/// (`Booting`), so a transient bus glitch does not kill a healthy boot. +/// A lapsed window is a timeout with no final read: a device that has not +/// reported cannot be judged on a race. +pub struct CheckpointWalk { + reader: R, + checkpoints: &'static [BootCheckpoint], + phase: Phase, +} + +enum Phase { + Idle, + Armed, + Walking { cursor: usize, deadline_millis: u64 }, +} + +impl CheckpointWalk { + /// Binds a reader and its checkpoint list into a walk. The checkpoints + /// are walked in declaration order; each one's signal is resolved by + /// the reader. + /// + /// # Panics + /// + /// Panics if `checkpoints` is empty. + pub fn new(reader: R, checkpoints: &'static [BootCheckpoint]) -> Self { + assert!(!checkpoints.is_empty(), "checkpoint list must not be empty"); + Self { + reader, + checkpoints, + phase: Phase::Idle, + } + } + + /// Mutable access to the reader. + pub fn reader_mut(&mut self) -> &mut R { + &mut self.reader + } +} + +impl, G> BootWatch for CheckpointWalk { + fn arm(&mut self) { + self.phase = Phase::Armed; + } + + fn poll(&mut self, now_millis: u64) -> WalkVerdict { + if let Phase::Armed = self.phase { + let timeout_millis = self.checkpoints[0].timeout().as_millis() as u64; + let deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: 0, + deadline_millis: deadline, + }; + } + + let (cursor, deadline) = match self.phase { + Phase::Walking { + cursor, + deadline_millis, + } => (cursor, deadline_millis), + _ => { + return WalkVerdict::Waiting { + deadline_millis: u64::MAX, + } + } + }; + + if now_millis >= deadline { + self.phase = Phase::Idle; + return WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::TimedOut, + }; + } + + let status = self + .reader + .read(self.checkpoints[cursor].signal()) + .unwrap_or(BootStatus::Booting); + + match status { + BootStatus::Booting => WalkVerdict::Waiting { + deadline_millis: deadline, + }, + BootStatus::Booted => { + let next = cursor + 1; + if next == self.checkpoints.len() { + self.phase = Phase::Idle; + WalkVerdict::Complete + } else { + let timeout_millis = self.checkpoints[next].timeout().as_millis() as u64; + let new_deadline = now_millis.saturating_add(timeout_millis); + self.phase = Phase::Walking { + cursor: next, + deadline_millis: new_deadline, + }; + WalkVerdict::Waiting { + deadline_millis: new_deadline, + } + } + } + BootStatus::FailedRetriable => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceRetriable, + } + } + BootStatus::FailedFatal => { + self.phase = Phase::Idle; + WalkVerdict::Failed { + checkpoint: self.checkpoints[cursor].name(), + cause: FailureCause::DeviceFatal, + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use core::time::Duration; + + const BL1: BootCheckpoint = BootCheckpoint::new("bl1", 1, Duration::from_millis(100)); + const KERNEL: BootCheckpoint = BootCheckpoint::new("kernel", 2, Duration::from_millis(200)); + const CHECKPOINTS: &[BootCheckpoint] = &[BL1, KERNEL]; + + // A progress-register reader: signal N is Booted once progress >= N. + // Mirrors the SocReader archetype in the evidence tests. + struct ProgressReader { + level: u8, + fault: Option, + fail_read: bool, + } + + impl ProgressReader { + fn new() -> Self { + Self { + level: 0, + fault: None, + fail_read: false, + } + } + } + + #[derive(Debug)] + struct ReadFault; + + impl core::fmt::Display for ReadFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("read fault") + } + } + + impl core::error::Error for ReadFault {} + + impl EvidenceReader for ProgressReader { + type Error = ReadFault; + + fn read(&mut self, signal: &u8) -> Result { + if self.fail_read { + return Err(ReadFault); + } + if let Some(fault) = self.fault { + return Ok(fault); + } + Ok(if self.level >= *signal { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } + } + + fn walk() -> CheckpointWalk { + CheckpointWalk::new(ProgressReader::new(), CHECKPOINTS) + } + + // ── Happy path ────────────────────────────────────────────────────── + + #[test] + fn two_checkpoint_walk_completes_when_both_pass() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + }, + "bl1 passed, now waiting on kernel" + ); + + let v = w.poll(0); + assert_eq!(v, WalkVerdict::Complete); + } + + #[test] + fn single_checkpoint_walk_completes_in_one_poll() { + let one = &[BL1] as &[_]; + // leak to get 'static + let one: &'static [BootCheckpoint] = Box::leak(one.to_vec().into_boxed_slice()); + let mut w = CheckpointWalk::new(ProgressReader::new(), one); + w.reader_mut().level = 1; + w.arm(); + + assert_eq!(w.poll(0), WalkVerdict::Complete); + } + + #[test] + fn progress_between_polls_advances_the_walk() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + w.reader_mut().level = 1; + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + }, + "bl1 passed at t=50, kernel deadline = 50 + 200" + ); + + w.reader_mut().level = 2; + let v = w.poll(100); + assert_eq!(v, WalkVerdict::Complete); + } + + // ── Timeout ───────────────────────────────────────────────────────── + + #[test] + fn first_checkpoint_times_out_when_device_is_silent() { + let mut w = walk(); + w.arm(); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + } + ); + + let v = w.poll(100); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + } + ); + } + + #[test] + fn second_checkpoint_times_out_after_first_passes() { + let mut w = walk(); + w.arm(); + + w.reader_mut().level = 1; + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + + let v = w.poll(200); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::TimedOut, + } + ); + } + + // ── Device-reported failures ──────────────────────────────────────── + + #[test] + fn retriable_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedRetriable); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceRetriable, + } + ); + } + + #[test] + fn fatal_failure_ends_the_walk_early() { + let mut w = walk(); + w.arm(); + w.reader_mut().fault = Some(BootStatus::FailedFatal); + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Read errors ───────────────────────────────────────────────────── + + #[test] + fn read_error_treated_as_silence() { + let mut w = walk(); + w.arm(); + w.reader_mut().fail_read = true; + + let v = w.poll(0); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 100 + }, + "bus glitch does not kill a healthy boot" + ); + + // Clear the fault and advance: the walk continues. + w.reader_mut().fail_read = false; + w.reader_mut().level = 2; + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 210 + } + ); + assert_eq!(w.poll(10), WalkVerdict::Complete); + } + + // ── arm() rewinds ─────────────────────────────────────────────────── + + #[test] + fn arm_rewinds_to_the_first_checkpoint() { + let mut w = walk(); + w.reader_mut().level = 2; + w.arm(); + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: 200 + } + ); + assert_eq!(w.poll(0), WalkVerdict::Complete); + + // Re-arm: back to checkpoint 0. + w.reader_mut().level = 0; + w.arm(); + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + }, + "fresh deadline from the re-arm" + ); + } + + #[test] + fn arm_mid_walk_restarts_from_the_beginning() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // passes bl1, now at kernel + + w.arm(); // restart + w.reader_mut().level = 0; + let v = w.poll(500); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 600 + }, + "restarted at bl1 with a fresh deadline" + ); + } + + // ── Unarmed / idle ───────────────────────────────────────────────── + + #[test] + fn unarmed_walk_waits_indefinitely() { + let w = walk(); + // Deliberately not calling arm(). + let mut w = w; + assert_eq!( + w.poll(0), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + #[test] + fn idle_after_terminal_waits_indefinitely() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + let v = w.poll(100); // now >= deadline -> TimedOut + assert!(matches!(v, WalkVerdict::Failed { .. })); + + assert_eq!( + w.poll(200), + WalkVerdict::Waiting { + deadline_millis: u64::MAX + } + ); + } + + // ── Deadline arithmetic ───────────────────────────────────────────── + + #[test] + fn deadline_is_relative_to_first_poll_not_arm() { + let mut w = walk(); + w.arm(); + // First poll at t=1000: deadline should be 1000 + 100, not 0 + 100. + let v = w.poll(1000); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 1100 + } + ); + } + + #[test] + fn next_checkpoint_deadline_is_relative_to_the_passing_poll() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + + // bl1 passes at t=50, kernel deadline = 50 + 200. + let v = w.poll(50); + assert_eq!( + v, + WalkVerdict::Waiting { + deadline_millis: 250 + } + ); + } + + // ── Decision 3: lapsed window = timeout, no last-chance read ──────── + + #[test] + fn booted_at_expiry_is_still_timeout() { + let mut w = walk(); + w.arm(); + w.poll(0); // Armed -> Walking, deadline = 100 + + w.reader_mut().level = 1; + let v = w.poll(100); // device ready, but window already lapsed + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "bl1", + cause: FailureCause::TimedOut, + }, + "no last-chance read: lapsed is lapsed" + ); + } + + // ── Fault at a later checkpoint names it correctly ──────────────── + + #[test] + fn device_fault_at_second_checkpoint_names_it() { + let mut w = walk(); + w.reader_mut().level = 1; + w.arm(); + w.poll(0); // bl1 passes, now at kernel + + w.reader_mut().fault = Some(BootStatus::FailedFatal); + let v = w.poll(10); + assert_eq!( + v, + WalkVerdict::Failed { + checkpoint: "kernel", + cause: FailureCause::DeviceFatal, + } + ); + } + + // ── Construction ──────────────────────────────────────────────────── + + #[test] + #[should_panic(expected = "checkpoint list must not be empty")] + fn empty_checkpoints_panic_at_construction() { + let empty: &'static [BootCheckpoint] = &[]; + CheckpointWalk::new(ProgressReader::new(), empty); + } +} diff --git a/services/orchestrator/config/src/lib.rs b/services/orchestrator/config/src/lib.rs index dae78ac7..3931ed5b 100644 --- a/services/orchestrator/config/src/lib.rs +++ b/services/orchestrator/config/src/lib.rs @@ -17,16 +17,46 @@ /// re-resets the device and re-runs the whole walk, so budgets are /// per boot attempt and owned by the orchestrator state machine. /// -/// The signal is a board-defined id — the schema attaches no meaning to +/// The signal is a board-defined id, the schema attaches no meaning to /// it and names no signal kinds. Each board defines its own vocabulary (a /// small enum: a GPIO line, a progress-register threshold, a message-path /// readiness) and gives it meaning in its `EvidenceReader`. The id is a /// defunctionalized evidence check: data in the table instead of a -/// function, so the table stays printable, comparable, const-checkable — +/// function, so the table stays printable, comparable, const-checkable, /// and could one day be generated instead of written. /// /// Fields are private so a checkpoint that violates the schema is /// unrepresentable: [`new`](Self::new) is the only way in, and it checks. +/// +/// # Example: three GPIO checkpoints +/// +/// A BMC behind three GPIO ready lines (bl1 on pin 4, kernel on pin 5, +/// service on pin 6), all on the same SGPIOM bank. Each signal variant +/// maps to one `GpioBootMonitor` in the board's `EvidenceReader`, and +/// the walker (`CheckpointWalk`) walks them in declaration order. +/// +/// ```ignore +/// #[derive(Debug, Clone, Copy)] +/// enum BmcSignal { Bl1, Kernel, Service } +/// +/// const BMC: DeviceConfig = DeviceConfig::new( +/// "bmc", 0, +/// &[ +/// BootCheckpoint::new("bl1", BmcSignal::Bl1, Duration::from_millis(500)), +/// BootCheckpoint::new("kernel", BmcSignal::Kernel, Duration::from_secs(5)), +/// BootCheckpoint::new("service", BmcSignal::Service, Duration::from_secs(30)), +/// ], +/// ); +/// +/// // Pin binding at bring-up: one GpioBootMonitor per signal. +/// let bl1 = GpioBootMonitor::new(&sgpiom, Mask(1 << 4), ActivePolarity::ActiveHigh); +/// let kernel = GpioBootMonitor::new(&sgpiom, Mask(1 << 5), ActivePolarity::ActiveHigh); +/// let service = GpioBootMonitor::new(&sgpiom, Mask(1 << 6), ActivePolarity::ActiveHigh); +/// +/// // The board's EvidenceReader dispatches signal to monitor. +/// // See EvidenceReader's docs for the full impl pattern. +/// let bmc_walk = CheckpointWalk::new(bmc_reader, BMC.checkpoints()); +/// ``` #[derive(Debug, Clone, Copy)] pub struct BootCheckpoint { name: &'static str, diff --git a/target/ast10x0/boot_evidence/BUILD.bazel b/target/ast10x0/boot_evidence/BUILD.bazel new file mode 100644 index 00000000..7745618f --- /dev/null +++ b/target/ast10x0/boot_evidence/BUILD.bazel @@ -0,0 +1,22 @@ +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "ast10x0_boot_evidence", + srcs = ["src/lib.rs"], + edition = "2024", + visibility = ["//visibility:public"], + deps = [ + "//hal/blocking", + "//services/orchestrator/adapters/hal:orchestrator_hal_adapters", + "//services/orchestrator/capabilities:orchestrator_capabilities", + "//services/orchestrator/config:orchestrator_config", + ], +) + +rust_test( + name = "ast10x0_boot_evidence_test", + crate = ":ast10x0_boot_evidence", +) diff --git a/target/ast10x0/boot_evidence/src/lib.rs b/target/ast10x0/boot_evidence/src/lib.rs new file mode 100644 index 00000000..2c92e828 --- /dev/null +++ b/target/ast10x0/boot_evidence/src/lib.rs @@ -0,0 +1,177 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! Boot-evidence wiring for the AST10x0 eRoT board. +//! +//! Binds the board's boot-checkpoint signals to concrete hardware: each +//! signal in the device table maps to one `GpioBootMonitor` on a +//! specific SGPIOM pin. The orchestrator never learns which pin belongs +//! to which device; this crate makes that binding once. + +#![no_std] + +use openprot_hal_blocking::gpio_port::{ActivePolarity, GpioPort}; +use orchestrator_capabilities::{BootStatus, EvidenceReader}; +use orchestrator_hal_adapters::{GpioBootMonitor, MonitorError}; + +/// Bit offset of BL1's ready line in SGPIOM bank EH (pin 42, bank base 32). +/// The typed `SgpiomMask` binding lives in the platform driver; this crate +/// records the offset so the table and the wiring agree on a single source. +pub const BL1_PIN_OFFSET: u32 = 10; + +/// BL1 ready line is active-high: the BMC asserts the pin when bl1 is up. +pub const BL1_POLARITY: ActivePolarity = ActivePolarity::ActiveHigh; + +/// Boot-checkpoint signal vocabulary for the BMC. Each variant names +/// one checkpoint in the device table; the `EvidenceReader` impl +/// dispatches it to the right `GpioBootMonitor`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BmcSignal { + /// First-stage bootloader ready (SGPIOM pin 42, bank EH bit 10). + Bl1, +} + +/// Reads the BMC's boot evidence off the SGPIOM. One `GpioBootMonitor` +/// per checkpoint signal, all sharing the same bank port. +pub struct BmcBootReader<'a, P: GpioPort> { + bl1: GpioBootMonitor<'a, P>, +} + +impl<'a, P: GpioPort> BmcBootReader<'a, P> { + /// Binds the reader to its monitors. Each monitor is constructed by + /// the platform driver at bring-up from a `(port, pin, polarity)` + /// triple. + pub fn new(bl1: GpioBootMonitor<'a, P>) -> Self { + Self { bl1 } + } +} + +impl EvidenceReader for BmcBootReader<'_, P> +where + P::Error: 'static, +{ + type Error = MonitorError; + + fn read(&mut self, signal: &BmcSignal) -> Result { + match signal { + BmcSignal::Bl1 => self.bl1.boot_status(), + } + } +} + +/// The BMC's device table entry. One checkpoint: bl1 on SGPIOM pin 42, +/// 500 ms window. +pub const BMC_DEVICE: orchestrator_config::DeviceConfig = + orchestrator_config::DeviceConfig::new( + "bmc", + 0, + &[orchestrator_config::BootCheckpoint::new( + "bl1", + BmcSignal::Bl1, + core::time::Duration::from_millis(500), + )], + ); + +#[cfg(test)] +mod tests { + use super::*; + use openprot_hal_blocking::gpio_port::{GpioError, GpioErrorKind, GpioErrorType, PinMask}; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct Mask(u32); + + impl PinMask for Mask { + fn empty() -> Self { + Self(0) + } + fn all() -> Self { + Self(u32::MAX) + } + fn is_empty(&self) -> bool { + self.0 == 0 + } + fn contains(&self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + fn union(&self, other: Self) -> Self { + Self(self.0 | other.0) + } + fn intersection(&self, other: Self) -> Self { + Self(self.0 & other.0) + } + fn toggle(&self) -> Self { + Self(!self.0) + } + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct MockError; + + impl GpioError for MockError { + fn kind(&self) -> GpioErrorKind { + GpioErrorKind::HardwareFailure + } + } + + struct MockPort { + input: Mask, + } + + impl GpioErrorType for MockPort { + type Error = MockError; + } + + impl GpioPort for MockPort { + type Config = (); + type Mask = Mask; + + fn read_input(&self) -> Result { + Ok(self.input) + } + fn configure(&mut self, _: Mask, _: ()) -> Result<(), MockError> { + panic!("reader must not configure pins"); + } + fn set_reset(&mut self, _: Mask, _: Mask) -> Result<(), MockError> { + panic!("reader must not drive outputs"); + } + fn toggle(&mut self, _: Mask) -> Result<(), MockError> { + panic!("reader must not drive outputs"); + } + } + + const BL1_PIN: Mask = Mask(1 << BL1_PIN_OFFSET); + + #[test] + fn bl1_reads_booted_when_pin42_is_high() { + let port = MockPort { + input: Mask(1 << 10), + }; + let mon = GpioBootMonitor::new(&port, BL1_PIN, BL1_POLARITY); + let mut reader = BmcBootReader::new(mon); + + assert_eq!( + reader.read(&BmcSignal::Bl1).expect("read failed"), + BootStatus::Booted, + ); + } + + #[test] + fn bl1_reads_booting_when_pin42_is_low() { + let port = MockPort { input: Mask(0) }; + let mon = GpioBootMonitor::new(&port, BL1_PIN, BL1_POLARITY); + let mut reader = BmcBootReader::new(mon); + + assert_eq!( + reader.read(&BmcSignal::Bl1).expect("read failed"), + BootStatus::Booting, + ); + } + + #[test] + fn bmc_device_table_has_one_bl1_checkpoint() { + assert_eq!(BMC_DEVICE.name(), "bmc"); + assert_eq!(BMC_DEVICE.checkpoints().len(), 1); + assert_eq!(BMC_DEVICE.checkpoints()[0].name(), "bl1"); + assert_eq!(*BMC_DEVICE.checkpoints()[0].signal(), BmcSignal::Bl1); + } +} diff --git a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel index d8d6a98f..e80197d9 100644 --- a/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel +++ b/target/ast10x0/tests/orchestrator/runtime/BUILD.bazel @@ -61,6 +61,8 @@ rust_app( tags = ["kernel"], target_compatible_with = TARGET_COMPATIBLE_WITH, deps = [ + "//services/orchestrator/adapters/walk:orchestrator_checkpoint_walk", + "//services/orchestrator/capabilities:orchestrator_capabilities", "//services/orchestrator/config:orchestrator_config", "//services/orchestrator/server:orchestrator_server", "//services/orchestrator/sm:orchestrator_sm", diff --git a/target/ast10x0/tests/orchestrator/runtime/main.rs b/target/ast10x0/tests/orchestrator/runtime/main.rs index ce2f3ad5..6fbce6e5 100644 --- a/target/ast10x0/tests/orchestrator/runtime/main.rs +++ b/target/ast10x0/tests/orchestrator/runtime/main.rs @@ -1,28 +1,20 @@ // Licensed under the Apache-2.0 license // SPDX-License-Identifier: Apache-2.0 -//! Orchestrator integration QEMU test: all four subcomponents wired end to end -//! under the kernel — the pure core ([`Orchestrator`], `orchestrator-sm`), the -//! server runtime ([`BootWatchdogs`], `orchestrator-server`) which wraps the -//! watchdog keeper (`orchestrator-timer`), and the board device table -//! ([`DeviceConfig`], `orchestrator-config`). +//! Orchestrator integration QEMU test: the pure core (`Orchestrator`, +//! `orchestrator-sm`), the server runtime (`BootWatchdogs`, +//! `orchestrator-server`), the device table (`DeviceConfig`, +//! `orchestrator-config`), and the checkpoint walker (`CheckpointWalk`, +//! `orchestrator-checkpoint-walk`) wired end to end under the kernel. //! -//! The runtime owns the clock and the mapping, so the platform driver stays -//! thin: -//! - boot windows come from the device table ([`BootCheckpoint::timeout`]); -//! the platform driver only converts `core::time::Duration` to the -//! kernel's [`Duration`] at the arm site. -//! - [`BootWatchdogs::arm_boot`] takes that *relative* window; the runtime -//! computes the absolute deadline. -//! - [`BootWatchdogs::wait_deadline`] is handed straight to `object_wait`. -//! - [`BootWatchdogs::poll_expired`] yields the `Event`s the core consumes — -//! no mapping in the platform driver. +//! Scenarios 1-2 exercise `CheckpointWalk`: the walk judges per-checkpoint +//! windows via `poll(now_millis)` and an `EvidenceReader`, while the runtime +//! uses the walk's `deadline_millis` as the `object_wait` argument. The walk +//! owns the verdict; the runtime owns the clock and wake scheduling. //! -//! Coverage: the *inner checkpoint walk* (`bl1` → `kernel`, re-armed through the -//! runtime) for a single component, the *outer component walk* across a -//! multi-component chain (nearest-of-many deadlines, correct-id recovery), and -//! the commit watchdog. The interrupt object (IRQ 44, self-fired) stands in for -//! a component reaching a checkpoint. +//! Scenarios 3-4 exercise `BootWatchdogs` multiplexing across a +//! multi-component chain (nearest-of-many deadlines, correct-id recovery). +//! Scenario 5 covers the commit watchdog. #![no_main] #![no_std] @@ -33,9 +25,11 @@ use openprot_orchestrator_sm::{ Chain, ComponentAttrs, ComponentId, Effect, EffectError, Event, Orchestrator, Platform, PowerOnResult, State, }; +use orchestrator_capabilities::{BootStatus, BootWatch, EvidenceReader, WalkVerdict}; +use orchestrator_checkpoint_walk::CheckpointWalk; use orchestrator_config::{BootCheckpoint, DeviceConfig}; use pw_status::{Error, Result}; -use userspace::time::Duration; +use userspace::time::{Clock, Duration, Instant, SystemClock}; use userspace::{entry, syscall}; /// The components this test supervises. @@ -56,13 +50,14 @@ type Watchdogs = BootWatchdogs; /// The device table: per-checkpoint windows, exactly as a board would declare /// them. Two checkpoints so the inner walk exercises re-arm-on-progress -/// (`bl1` then `kernel`). +/// (`bl1` then `kernel`). Signal ids are progress thresholds: the reader +/// reports `Booted` once its internal level reaches the threshold. const SOC: DeviceConfig = DeviceConfig::new( "soc", 0, &[ - BootCheckpoint::new("bl1", 0, core::time::Duration::from_millis(50)), - BootCheckpoint::new("kernel", 0, core::time::Duration::from_millis(50)), + BootCheckpoint::new("bl1", 1, core::time::Duration::from_millis(500)), + BootCheckpoint::new("kernel", 2, core::time::Duration::from_millis(500)), ], ); @@ -72,6 +67,32 @@ fn window(timeout: core::time::Duration) -> Duration { Duration::from_millis(timeout.as_millis() as u64) } +/// Progress-register reader for the walk: signal N is `Booted` once +/// `level >= N`. Mirrors the SocReader archetype in the evidence tests. +struct ProgressReader { + level: u8, +} + +impl EvidenceReader for ProgressReader { + type Error = core::convert::Infallible; + + fn read(&mut self, signal: &u8) -> core::result::Result { + Ok(if self.level >= *signal { + BootStatus::Booted + } else { + BootStatus::Booting + }) + } +} + +fn ticks_to_millis(ticks: u64) -> u64 { + ticks * 1000 / SystemClock::TICKS_PER_SEC +} + +fn millis_to_ticks(millis: u64) -> u64 { + millis * SystemClock::TICKS_PER_SEC / 1000 +} + /// A fake [`Platform`] for the run loop. It records the `ReleaseReset(id)` /// effects that open each component's boot supervision; every other effect is /// accepted so the core can settle. @@ -125,47 +146,50 @@ fn drive_releases(core: &mut Core, plat: &mut FakePlatform, ids: &[ComponentId]) Ok(()) } -/// Run one component's inner checkpoint walk through the runtime and return its -/// single terminal event. `reached` simulates the device: it fires its progress -/// signal for the first `reached` checkpoints, then goes quiet — so -/// `reached == len` boots, anything less times out at checkpoint `reached`. -fn checkpoint_walk( - wd: &mut Watchdogs, +/// Run one component's inner checkpoint walk through `CheckpointWalk` and +/// return its terminal verdict as an event. `reached` simulates the device: +/// the reader's progress advances for the first `reached` checkpoints, then +/// goes quiet, so `reached == len` boots and anything less times out. +/// +/// The walk judges per-checkpoint windows; the runtime (`object_wait`) just +/// sleeps until the walk's deadline or a device signal. No `BootWatchdogs` +/// are involved: the walk owns the verdict, the kernel clock owns the wake. +fn walk_device( + walk: &mut CheckpointWalk, id: ComponentId, - checkpoints: &[BootCheckpoint], reached: usize, ) -> Result { + walk.arm(); let mut k = 0usize; - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; - loop { - // Simulated device reaching checkpoint `k`: latch its progress signal - // before the wait (interrupt objects hold it pending, so no race). - if k < reached { - syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; - } - let deadline = wd.wait_deadline(); - match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { - Ok(wait) => { - if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { - return Err(Error::Internal); + loop { + let now = ticks_to_millis(SystemClock::now().ticks()); + match walk.poll(now) { + WalkVerdict::Waiting { deadline_millis } => { + // Simulate device progress after the poll returned Waiting, + // so every trigger pairs with a wait+ack below. + if k < reached { + walk.reader_mut().level = (k + 1) as u8; + syscall::debug_trigger_interrupt(constants::BOOT_PROGRESS)?; } - syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; - k += 1; - if k == checkpoints.len() { - wd.cancel_boot(id); - return Ok(Event::Booted(id)); + + let deadline = Instant::from_ticks(millis_to_ticks(deadline_millis)); + match syscall::object_wait(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS, deadline) { + Ok(wait) => { + if !wait.pending_signals.contains(signals::BOOT_PROGRESS) { + return Err(Error::Internal); + } + syscall::interrupt_ack(handle::BOOT_SIGNAL, signals::BOOT_PROGRESS)?; + k += 1; + } + Err(Error::DeadlineExceeded) => { + // Deadline lapsed; re-poll and the walk will judge timeout. + } + Err(e) => return Err(e), } - // Forward progress: re-arm the next checkpoint through the runtime. - wd.arm_boot(id, window(checkpoints[k].timeout())) - .map_err(|_| Error::ResourceExhausted)?; - } - Err(Error::DeadlineExceeded) => { - // The window lapsed: the runtime already mapped it to an `Event`. - return wd.poll_expired().ok_or(Error::Internal); } - Err(e) => return Err(e), + WalkVerdict::Complete => return Ok(Event::Booted(id)), + WalkVerdict::Failed { .. } => return Ok(Event::Timeout(id)), } } } @@ -192,14 +216,12 @@ fn confirm(wd: &mut Watchdogs, id: ComponentId) -> Result<()> { } /// Inner walk, happy path: a single component passes every checkpoint (windows -/// from the device table, re-armed through the runtime), the walk yields -/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op — the -/// watchdog was retired by the confirmation. +/// from the device table, judged by `CheckpointWalk`), the walk yields +/// `Booted`, and the core stays `Ready`. A late `Timeout` is then a no-op. fn scenario_checkpoint_confirmed() -> Result<()> { pw_log::info!("scenario 1: checkpoint walk confirmed"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; if core.state() != State::Ready { @@ -207,7 +229,8 @@ fn scenario_checkpoint_confirmed() -> Result<()> { return Err(Error::Internal); } - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), SOC.checkpoints().len())?; + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, SOC.checkpoints().len())?; if terminal != Event::Booted(C0) { pw_log::error!("scenario 1: walk did not confirm boot"); return Err(Error::Internal); @@ -218,7 +241,7 @@ fn scenario_checkpoint_confirmed() -> Result<()> { return Err(Error::Internal); } - // The watchdog is retired: a stale timeout must not re-open recovery. + // A stale timeout must not re-open recovery. core.dispatch(&mut plat, Event::Timeout(C0)); if core.state() != State::Ready { pw_log::error!("scenario 1: stale timeout re-opened recovery"); @@ -230,16 +253,17 @@ fn scenario_checkpoint_confirmed() -> Result<()> { } /// Inner walk, timeout path: the device never signals, the first checkpoint's -/// window lapses, the runtime surfaces `Timeout`, and the core recovers. +/// window lapses (judged by `CheckpointWalk`, not by `BootWatchdogs`), the +/// walk surfaces `Timeout`, and the core recovers. fn scenario_checkpoint_timeout() -> Result<()> { pw_log::info!("scenario 2: checkpoint walk timeout drives recovery"); let mut core = new_core(&[C0])?; let mut plat = FakePlatform::new(); - let mut wd = Watchdogs::new(); drive_releases(&mut core, &mut plat, &[C0])?; - let terminal = checkpoint_walk(&mut wd, C0, SOC.checkpoints(), 0)?; + let mut walk = CheckpointWalk::new(ProgressReader { level: 0 }, SOC.checkpoints()); + let terminal = walk_device(&mut walk, C0, 0)?; if terminal != Event::Timeout(C0) { pw_log::error!("scenario 2: walk did not time out"); return Err(Error::Internal);