From dcef6f1f2fe916bf44abd39c786acff36410eac5 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 3 Aug 2026 10:34:34 +0200 Subject: [PATCH 01/47] WIP: I/O API + simulator --- Cargo.lock | 4 + crates/runtime-core/Cargo.toml | 4 +- crates/runtime-core/src/io/mod.rs | 148 ++++++++++++ crates/runtime-core/src/lib.rs | 2 + crates/runtime-core/src/sim/io/fs.rs | 154 ++++++++++++ crates/runtime-core/src/sim/io/mod.rs | 261 +++++++++++++++++++++ crates/runtime-core/src/sim/io/op.rs | 322 ++++++++++++++++++++++++++ crates/runtime-core/src/sim/mod.rs | 1 + crates/runtime/Cargo.toml | 4 + crates/runtime/src/io.rs | 2 + crates/runtime/src/io/tokio.rs | 163 +++++++++++++ crates/runtime/src/lib.rs | 2 + 12 files changed, 1066 insertions(+), 1 deletion(-) create mode 100644 crates/runtime-core/src/io/mod.rs create mode 100644 crates/runtime-core/src/sim/io/fs.rs create mode 100644 crates/runtime-core/src/sim/io/mod.rs create mode 100644 crates/runtime-core/src/sim/io/op.rs create mode 100644 crates/runtime/src/io.rs create mode 100644 crates/runtime/src/io/tokio.rs diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..0289ef17a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8333,7 +8333,9 @@ dependencies = [ "futures", "libc", "spacetimedb-runtime-core", + "static_assertions", "tokio", + "windows-sys 0.61.2", ] [[package]] @@ -8341,7 +8343,9 @@ name = "spacetimedb-runtime-core" version = "2.8.3" dependencies = [ "async-task", + "futures-channel", "spin", + "zerocopy", ] [[package]] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index a3369a69f89..6ac037162dd 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -11,8 +11,10 @@ workspace = true [features] default = [] -sim = ["dep:async-task", "dep:spin"] +sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } +futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } +zerocopy = "0.8" diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs new file mode 100644 index 00000000000..b4ff3744916 --- /dev/null +++ b/crates/runtime-core/src/io/mod.rs @@ -0,0 +1,148 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +/// Size in bytes of a disk sector. +pub const SECTOR_SIZE: usize = 4096; + +/// Types that can be safely converted to and from sector-aligned byte slices. +pub trait AlignedBytes: Sized { + /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the + /// right aligment. + /// + /// NOTE: Associated constants are evaluated lazily -- add a free + /// + /// `const _: () = ::ASSERT_VALID_LAYOUT;` + /// + /// for each `T` that is supposed to be used as an `AlignedBytes`. + const ASSERT_VALID_LAYOUT: () = { + assert!(align_of::() == SECTOR_SIZE); + assert!(size_of::().is_multiple_of(SECTOR_SIZE)); + }; + + /// Reinterpret `self` as a byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes(&self) -> &[u8]; + + /// Reinterpret `self` as a mutable byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes_mut(&mut self) -> &mut [u8]; + + /// Reinterpret a byte slice as `Self`. + /// + /// The slice must be of length `size_of::()`. + /// + /// NOTE: Any slice of the right size, but consisting of only `0` (zero) + /// bytes can be converted to `Self`. It is the caller's responsibility to + /// validate the returned type as per the application's invariants. + /// + /// # Panics + /// + /// Panics if `b.len() != size_of::()`. + fn from_bytes(b: &[u8]) -> Self; +} + +impl AlignedBytes for T { + fn as_bytes(&self) -> &[u8] { + ::as_bytes(self) + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + ::as_mut_bytes(self) + } + + fn from_bytes(b: &[u8]) -> Self { + Self::read_from_bytes(b).unwrap() + } +} + +/// An error `E`, along with auxiliary data `T`. +/// +/// `T` is usually a buffer of type [AlignedBytes], whose ownership is +/// transferred back to the caller when an error occurs. +/// +/// As this type signifies an error condition, the contents of `T` are +/// unspecified. +#[derive(Debug)] +pub struct ErrorWith { + pub error: E, + pub with: T, +} + +/// The canonical, low-level I/O API. +/// +/// Currently only supports file I/O, but eventually all I/O performed by +/// SpacetimeDB should go through this trait. +/// +/// Intended to support implementations based on `io-uring`, which means that +/// buffer ownership is transferred to the I/O engine while reading or writing. +/// +/// Implementations should be `!Send`, i.e. all I/O happens on a single thread. +/// +/// File operations should never be mutually exclusive, and therefore expose a +/// `pwrite`/`pread`-style API. It is assumed that direct I/O (`O_DIRECT`) is +/// used, i.e. the kernel page cache is bypassed. The [AlignedBytes] type +/// ensures that the alignment requirements for direct I/O are met. +pub trait SpacetimeIO { + /// An open file handle. + /// + /// Like [std::fs::File], the file shall be closed when the last reference + /// to the handle is dropped. + /// + /// Unlike [std::fs::File], the file handle must be clone-able. + type Fd: Clone; + /// The error returned by methods of this trait. + /// + /// This should always be instantiated to [std::io::Error]. However, pending + /// [alloc_io], this type is not in `core`, which would prevent this crate + /// from being `no_std`. + /// + /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 + type Error; + + /// Open the file at `path`. + fn open_file(&self, path: &str) -> impl Future>; + + /// Create the file at `path` and allocate `len` bytes. + /// + /// Returns an error if the file already exists. + fn create_file(&self, path: &str, len: u64) -> impl Future>; + + /// Write `buf` to `fd` at `offset`. + /// + /// `offset` must be a multiple of [SECTOR_SIZE]. + /// + /// Behaves like `FileExt::write_all_at`, i.e. tries to write all bytes in + /// `buf`, potentially retrying on errors of kind interrupted, and returns + /// an error if that fails. + fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> impl Future>>; + + /// Read `size_of::()` bytes from `fd` at `offset` and interpret them at + /// type `B`. + /// + /// `offset` must be a multiple of [SECTOR_SIZE]. + /// + /// Behaves like `FileExt::read_exact_at`, i.e. attempts to read + /// `size_of::()` bytes, potentially retrying on errors of kind + /// interrupted, and returns an error if less than the required bytes could + /// be read. + fn read_exact_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> impl Future>>; + + /// Call `fsync(2)` on `fd`. + fn fsync(&self, fd: Self::Fd) -> impl Future>; + /// Call `fdatasync(2)` on `fd`. + fn fdatasync(&self, fd: Self::Fd) -> impl Future>; + + /// Allocate `additional` bytes for the file `fd`. + fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; +} diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index f7590ada98b..e35d042ea9a 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -7,3 +7,5 @@ extern crate std; #[cfg(feature = "sim")] pub mod sim; + +pub mod io; diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs new file mode 100644 index 00000000000..9c540f5e1b3 --- /dev/null +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -0,0 +1,154 @@ +use alloc::{collections::BTreeMap, rc::Rc}; +use core::{ + cell::{Cell, RefCell}, + cmp, +}; + +pub const PAGE_SIZE: usize = 4096; +const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Error { + UnalignedOffset, + UnalignedBuffer, + OffsetOverflow, +} + +pub type Result = core::result::Result; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct PageIndex(u64); + +impl PageIndex { + fn from_offset(offset: u64) -> Self { + assert!(offset.is_multiple_of(PAGE_SIZE_U64)); + Self(offset / PAGE_SIZE_U64) + } +} + +struct Page { + bytes: RefCell<[u8; PAGE_SIZE]>, +} + +impl Page { + fn zeroed() -> Self { + Self { + bytes: RefCell::new([0; PAGE_SIZE]), + } + } +} + +/// A memory-backed file. +/// +/// A [File] is backed by a sparse array of [Page]s. Missing pages are read as +/// zeroes. +/// +/// Read and write operations must be page-aligned. Only full pages can be read +/// or written. Writing a page is atomic. +#[derive(Clone)] +pub struct File { + pages: RefCell>>, + len: Cell, +} + +impl File { + pub(super) const fn new() -> Self { + Self { + pages: RefCell::new(BTreeMap::new()), + len: Cell::new(0), + } + } + + pub(super) const fn len(&self) -> u64 { + self.len.get() + } + + #[allow(unused)] + pub(super) const fn is_empty(&self) -> bool { + self.len.get() == 0 + } + + /// Change the file length. + /// + /// The new length must be page-aligned. + /// + /// Extending allocates pages eagerly as needed. Shrinking drops all pages + /// at or beyond the new EOF. + pub fn set_len(&self, new_len: u64) -> Result<()> { + use cmp::Ordering::*; + + if !new_len.is_multiple_of(PAGE_SIZE_U64) { + return Err(Error::UnalignedOffset); + } + let old_len = self.len.get(); + + match new_len.cmp(&old_len) { + Equal => {} + Greater => { + let first_new_page = old_len / PAGE_SIZE_U64; + let end_page = new_len / PAGE_SIZE_U64; + + for index in first_new_page..end_page { + self.get_or_allocate_page(PageIndex(index)); + } + + self.len.set(new_len); + } + Less => { + self.len.set(new_len); + + let first_removed = PageIndex::from_offset(new_len); + let removed = self.pages.borrow_mut().split_off(&first_removed); + drop(removed); + } + } + + Ok(()) + } + + /// Read one complete page. + pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { + if dst.len() != PAGE_SIZE { + return Err(Error::UnalignedBuffer); + } + + match self.get_page(PageIndex(index)) { + Some(page) => { + dst.copy_from_slice(&*page.bytes.borrow()); + } + None => { + dst.fill(0); + } + } + + Ok(()) + } + + /// Write one complete page. + pub fn write_page(&self, src: &[u8], index: u64) -> Result<()> { + if src.len() != PAGE_SIZE { + return Err(Error::UnalignedBuffer); + } + + let page = self.get_or_allocate_page(PageIndex(index)); + page.bytes.borrow_mut().copy_from_slice(src); + + let end = index + .checked_add(1) + .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) + .ok_or(Error::OffsetOverflow)?; + + self.len.set(cmp::max(self.len.get(), end)); + + Ok(()) + } + + fn get_page(&self, index: PageIndex) -> Option> { + self.pages.borrow().get(&index).cloned() + } + + fn get_or_allocate_page(&self, index: PageIndex) -> Rc { + let mut pages = self.pages.borrow_mut(); + Rc::clone(pages.entry(index).or_insert_with(|| Rc::new(Page::zeroed()))) + } +} diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs new file mode 100644 index 00000000000..670b68d7aa3 --- /dev/null +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -0,0 +1,261 @@ +use alloc::{ + boxed::Box, + collections::{BTreeMap, VecDeque}, + rc::Rc, +}; +use core::{ + cell::RefCell, + future::{poll_fn, Future}, + pin::Pin, + result::Result, + task::Poll, +}; +use futures_channel::oneshot; + +use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; + +mod fs; +mod op; + +pub use crate::io::SECTOR_SIZE; +pub use fs::File; + +#[derive(Debug)] +pub enum Error { + FileNotFound { path: Box }, + FileAlreadyExists { path: Box }, + ShortWrite { expected: usize, written: usize }, + UnexpectedEof { expected: usize, read: usize }, + Fs(fs::Error), +} + +impl From for Error { + fn from(e: fs::Error) -> Self { + Self::Fs(e) + } +} + +#[derive(Default)] +pub struct SimulatorIO { + inner: Rc>, +} + +impl SimulatorIO { + pub fn tick(&self) { + self.inner.borrow_mut().tick(); + } + + fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + self.inner.borrow_mut().submit(op(tx)); + rx + } + + // TODO: The sim runtime should be advancing I/O. Until it does, `tick()` + // whenever a result future is polled and returns pending. + async fn wait_for(&self, mut rx: oneshot::Receiver) -> Result { + poll_fn(|cx| match Pin::new(&mut rx).poll(cx) { + Poll::Ready(result) => Poll::Ready(result), + Poll::Pending => { + self.tick(); + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await + } + + async fn submit_and_wait( + &self, + op: impl FnOnce(oneshot::Sender) -> Box, + ) -> Result { + let rx = self.submit(op); + self.wait_for(rx).await + } +} + +impl SpacetimeIO for SimulatorIO { + type Fd = fs::File; + type Error = Error; + + async fn open_file(&self, path: &str) -> Result { + self.submit_and_wait(|tx| op::open_file(path, tx)) + .await + .expect("`open_file` future cancelled") + } + + async fn create_file(&self, path: &str, len: u64) -> Result { + self.submit_and_wait(|tx| op::create_file(path, len, tx)) + .await + .expect("`create_file` future cancelled") + } + + async fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let () = B::ASSERT_VALID_LAYOUT; + + if !offset.is_multiple_of(SECTOR_SIZE as _) { + self.submit_and_wait(|tx| { + op::ready( + Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }), + tx, + ) + }) + .await + .expect("`write_all_at` future cancelled") + } else { + let (tx, rx) = oneshot::channel(); + for op in op::write_at(fd, buf, offset, tx) { + self.inner.borrow_mut().submit(op); + } + self.wait_for(rx).await.expect("`write_all_at` future cancelled") + } + } + + async fn read_exact_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let () = B::ASSERT_VALID_LAYOUT; + + if !offset.is_multiple_of(SECTOR_SIZE as _) { + self.submit_and_wait(|tx| { + op::ready( + Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }), + tx, + ) + }) + .await + .expect("`read_exact_at` future cancelled") + } else { + let (tx, rx) = oneshot::channel(); + for op in op::read_at(fd, buf, offset, tx) { + self.inner.borrow_mut().submit(op); + } + self.wait_for(rx).await.expect("`read_exact_at` future cancelled") + } + } + + async fn fsync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { + Ok(()) + } + + async fn fdatasync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { + Ok(()) + } + + async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { + let len = self + .submit_and_wait(|tx| op::get_len(fd.clone(), tx)) + .await + .expect("`get_len` future cancelled")?; + self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) + .await + .expect("`set_len` future cancelled") + } +} + +#[derive(Default)] +struct SimulatorIOInner { + files: BTreeMap, fs::File>, + submissions: VecDeque>, + completions: VecDeque>, +} + +impl SimulatorIOInner { + fn tick(&mut self) { + if let Some(sqe) = self.submissions.pop_front() { + sqe.execute(&mut self.files, &mut self.completions); + } + if let Some(cqe) = self.completions.pop_front() { + cqe.complete(); + } + } + + fn submit(&mut self, op: Box) { + self.submissions.push_back(op); + } +} + +trait Submission { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ); +} + +trait Completion { + fn complete(self: Box); +} + +#[cfg(test)] +mod tests { + use crate::sim::Runtime; + + use super::*; + + #[test] + fn create_file() { + let mut rt = Runtime::new(1); + let io = SimulatorIO::default(); + + let fd = rt + .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) + .unwrap(); + assert_eq!(fd.len(), 2 * SECTOR_SIZE as u64); + } + + #[repr(C, align(4096))] + struct Buf([u8; 2 * SECTOR_SIZE]); + + impl AlignedBytes for Buf { + fn as_bytes(&self) -> &[u8] { + &self.0 + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 + } + + fn from_bytes(b: &[u8]) -> Self { + assert_eq!(b.len(), 2 * SECTOR_SIZE); + let mut buf = [0; 2 * SECTOR_SIZE]; + buf.copy_from_slice(b); + Self(buf) + } + } + + #[test] + fn write_read_roundtrip() { + let mut rt = Runtime::new(1); + let io = SimulatorIO::default(); + + let fd = rt + .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) + .unwrap(); + let mut buf = rt + .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) + .map_err(|ErrorWith { error, .. }| error) + .unwrap(); + buf.0.fill(0); + let buf = rt + .block_on(io.read_exact_at(fd, buf, 0)) + .map_err(|ErrorWith { error, .. }| error) + .unwrap(); + + assert!(buf.0.iter().all(|&b| b == 22)); + } +} diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs new file mode 100644 index 00000000000..921129d2536 --- /dev/null +++ b/crates/runtime-core/src/sim/io/op.rs @@ -0,0 +1,322 @@ +use alloc::{ + boxed::Box, + collections::{btree_map, BTreeMap, VecDeque}, + rc::Rc, +}; +use core::cell::RefCell; +use futures_channel::oneshot; + +use super::{fs, Completion, Error, Submission}; +use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; + +pub type WriteAtResult = Result>; +pub type ReadAtResult = Result>; + +pub fn write_at( + fd: fs::File, + buf: B, + offset: u64, + notify: oneshot::Sender>, +) -> impl Iterator> { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Rc::new(RefCell::new(PagedOpState { + buf: Some(buf), + notify: Some(notify), + remaining: page_count, + first_error: None, + })); + + (0..page_count).map(move |buf_page| { + let op = WritePage { + fd: fd.clone(), + file_page: first_page + buf_page, + buf_page, + state: state.clone(), + }; + + Box::new(op) as Box + }) +} + +pub fn read_at( + fd: fs::File, + buf: B, + offset: u64, + notify: oneshot::Sender>, +) -> impl Iterator> { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Rc::new(RefCell::new(PagedOpState { + buf: Some(buf), + notify: Some(notify), + remaining: page_count, + first_error: None, + })); + + (0..page_count).map(move |buf_page| { + let op = ReadPage { + fd: fd.clone(), + file_page: first_page + buf_page, + buf_page, + state: state.clone(), + }; + + Box::new(op) as Box + }) +} + +pub fn open_file(path: &str, notify: oneshot::Sender>) -> Box { + Box::new(OpenFile { + path: path.into(), + notify, + }) +} + +pub fn create_file(path: &str, len: u64, notify: oneshot::Sender>) -> Box { + Box::new(CreateFile { + path: path.into(), + len, + notify, + }) +} + +pub fn get_len(fd: fs::File, notify: oneshot::Sender>) -> Box { + Box::new(GetLen { fd, notify }) +} + +pub fn set_len(fd: fs::File, len: u64, notify: oneshot::Sender>) -> Box { + Box::new(SetLen { fd, len, notify }) +} + +struct GenericCompletion { + result: T, + notify: oneshot::Sender, +} + +fn completion(result: T, notify: oneshot::Sender) -> Box { + Box::new(GenericCompletion { result, notify }) +} + +impl Completion for GenericCompletion { + fn complete(self: Box) { + let Self { result, notify } = *self; + let _ = notify.send(result); + } +} + +struct Ready(Box); + +impl Submission for Ready { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self(completion) = *self; + completions.push_back(completion); + } +} + +pub fn ready(result: T, notify: oneshot::Sender) -> Box { + Box::new(Ready(completion(result, notify))) +} + +struct OpenFile { + path: Box, + notify: oneshot::Sender>, +} + +impl Submission for OpenFile { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { path, notify } = *self; + let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); + completions.push_back(completion(result, notify)); + } +} + +struct CreateFile { + path: Box, + len: u64, + notify: oneshot::Sender>, +} + +impl Submission for CreateFile { + fn execute( + self: Box, + files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { path, len, notify } = *self; + let result = (|| { + let file = match files.entry(path.clone()) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(_) => Err(Error::FileAlreadyExists { path }), + }?; + file.set_len(len)?; + Ok(file) + })(); + completions.push_back(completion(result, notify)); + } +} + +struct GetLen { + fd: fs::File, + notify: oneshot::Sender>, +} + +impl Submission for GetLen { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { fd, notify } = *self; + let result = Ok(fd.len()); + completions.push_back(completion(result, notify)); + } +} + +struct SetLen { + fd: fs::File, + len: u64, + notify: oneshot::Sender>, +} + +impl Submission for SetLen { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { fd, len, notify } = *self; + let result = fd.set_len(len).map_err(Error::from); + completions.push_back(completion(result, notify)); + } +} + +struct PagedOpState { + buf: Option, + notify: Option>>>, + remaining: usize, + first_error: Option, +} + +fn complete_page_op( + state: &Rc>>, + result: Result<(), fs::Error>, + completions: &mut VecDeque>, +) { + let complete = { + let mut state = state.borrow_mut(); + if let Err(e) = result + && state.first_error.is_none() + { + state.first_error.replace(e.into()); + } + assert!(state.remaining > 0); + state.remaining -= 1; + + state.remaining == 0 + }; + + if complete { + completions.push_back(Box::new(WriteCompletion { state: state.clone() })); + } +} + +struct WriteCompletion { + state: Rc>>, +} + +impl Completion for WriteCompletion { + fn complete(self: Box) { + let (notify, result) = { + let mut state = self.state.borrow_mut(); + + assert_eq!(state.remaining, 0); + + let buf = state.buf.take().expect("write completed more than once"); + let notify = state.notify.take().expect("write completed more than once"); + + let result = match state.first_error.take() { + None => Ok(buf), + Some(error) => Err(ErrorWith { error, with: buf }), + }; + + (notify, result) + }; + + let _ = notify.send(result); + } +} + +struct WritePage { + fd: fs::File, + file_page: usize, + buf_page: usize, + state: Rc>>, +} + +impl Submission for WritePage { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { + fd, + file_page, + buf_page, + state, + } = *self; + + let result = { + let state_ref = state.borrow(); + let buf = state_ref.buf.as_ref().expect("buffer went away"); + + let start = buf_page * SECTOR_SIZE; + let end = start + SECTOR_SIZE; + fd.write_page(&buf.as_bytes()[start..end], file_page as _) + }; + complete_page_op(&state, result, completions); + } +} + +struct ReadPage { + fd: fs::File, + file_page: usize, + buf_page: usize, + state: Rc>>, +} + +impl Submission for ReadPage { + fn execute( + self: Box, + _files: &mut BTreeMap, fs::File>, + completions: &mut VecDeque>, + ) { + let Self { + fd, + file_page, + buf_page, + state, + } = *self; + + let result = { + let mut state_ref = state.borrow_mut(); + let buf = state_ref.buf.as_mut().expect("buffer went away"); + + let start = buf_page * SECTOR_SIZE; + let end = start + SECTOR_SIZE; + fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) + }; + complete_page_op(&state, result, completions); + } +} diff --git a/crates/runtime-core/src/sim/mod.rs b/crates/runtime-core/src/sim/mod.rs index e2c231828a1..1a5a53a29bf 100644 --- a/crates/runtime-core/src/sim/mod.rs +++ b/crates/runtime-core/src/sim/mod.rs @@ -1,5 +1,6 @@ pub mod buggify; mod executor; +pub mod io; mod rng; pub mod time; diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index c8affea0f48..2b2cffc5317 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -13,6 +13,10 @@ workspace = true tokio.workspace = true spacetimedb-runtime-core = { workspace = true, optional = true } libc = { version = "0.2", optional = true } +static_assertions = "1.1" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [dev-dependencies] futures.workspace = true diff --git a/crates/runtime/src/io.rs b/crates/runtime/src/io.rs new file mode 100644 index 00000000000..f7c24fe029f --- /dev/null +++ b/crates/runtime/src/io.rs @@ -0,0 +1,2 @@ +mod tokio; +pub use tokio::TokioIO as Tokio; diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs new file mode 100644 index 00000000000..eb948ed7e1c --- /dev/null +++ b/crates/runtime/src/io/tokio.rs @@ -0,0 +1,163 @@ +use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; + +#[cfg(unix)] +use std::os::unix::fs::FileExt as _; +#[cfg(windows)] +use std::os::windows::fs::FileExt as _; + +use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use static_assertions::assert_not_impl_any; +use tokio::fs::OpenOptions; +use tokio::{runtime, task::spawn_blocking}; + +/// Implementation of [SpacetimeIO] that runs on a tokio runtime. +pub struct TokioIO { + // TODO: Should this be [runtime::Runtime]? + rt: runtime::Handle, + // Ensure I/O stays on a single thread. + _not_send: PhantomData>, +} + +impl TokioIO { + pub fn new(rt: runtime::Handle) -> Self { + Self { + rt, + _not_send: PhantomData, + } + } +} + +assert_not_impl_any!(TokioIO: Send); + +impl SpacetimeIO for TokioIO { + // NOTE: This operates on a [std::fs::File] handle instead of + // [tokio::fs::File] because `pwrite`/`pread`-style APIs are not available + // from tokio proper. As a consequence, operations on an open `Fd` use + // [spawn_blocking]. This is what [tokio::fs::File] does internally, while + // here we can avoid some locking. + type Fd = Arc; + type Error = io::Error; + + async fn open_file(&self, path: &str) -> Result { + let _rt = self.rt.enter(); + + let mut open_options = tokio::fs::File::options(); + open_options.read(true).write(true); + let file = open_with_direct_io(open_options, path).await?; + + Ok(Arc::new(file.into_std().await)) + } + + async fn create_file(&self, path: &str, len: u64) -> Result { + let _rt = self.rt.enter(); + + let mut open_options = tokio::fs::File::options(); + open_options.read(true).write(true).create_new(true); + let file = open_with_direct_io(open_options, path).await?; + file.set_len(len).await?; + + Ok(Arc::new(file.into_std().await)) + } + + async fn write_all_at( + &self, + fd: Self::Fd, + buf: B, + offset: u64, + ) -> Result> { + let _rt = self.rt.enter(); + asyncify(move || { + #[cfg(unix)] + let res = fd.write_all_at(buf.as_bytes(), offset); + #[cfg(windows)] + let res = fd.seek_write(buf.as_bytes(), offset); + + match res { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + } + }) + .await + } + + async fn read_exact_at( + &self, + fd: Self::Fd, + mut buf: B, + offset: u64, + ) -> Result> { + let _rt = self.rt.enter(); + asyncify(move || { + #[cfg(unix)] + let res = fd.read_exact_at(buf.as_bytes_mut(), offset); + #[cfg(windows)] + let res = fd.seek_read(buf.as_bytes_mut(), offset); + + match res { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + } + }) + .await + } + + async fn fsync(&self, fd: Self::Fd) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || fd.sync_all()).await + } + + async fn fdatasync(&self, fd: Self::Fd) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || fd.sync_data()).await + } + + async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { + let _rt = self.rt.enter(); + asyncify(move || { + let len = fd.metadata()?.len(); + fd.set_len(len + additional)?; + + Ok(()) + }) + .await + } +} + +async fn asyncify(f: F) -> R +where + F: FnOnce() -> R + Send + 'static, + R: Send + 'static, +{ + spawn_blocking(f).await.unwrap_or_else(|e| match e.try_into_panic() { + Ok(panic_payload) => std::panic::resume_unwind(panic_payload), + // A cancellation should not be possible, because we await the task. + Err(e) => panic!("unexpected error joining blocking task: {e}"), + }) +} + +#[cfg(all(unix, not(target_os = "macos")))] +async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { + options.custom_flags(libc::O_DIRECT).open(path).await +} + +#[cfg(target_os = "macos")] +async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { + let file = options.open(path).await?; + asyncify(move || { + let res = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; + if res == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(file) + } + }) + .await +} + +#[cfg(target_os = "windows")] +async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { + options + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) + .open(path) + .await +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index c6192e1b738..48400676009 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -53,6 +53,8 @@ pub enum Handle { Simulation(sim::Handle), } +pub mod io; + pub struct JoinHandle { inner: JoinHandleInner, } From 11dc4c7197ebc2bd2a948af0bb3fc6c5b4295b1a Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 4 Aug 2026 15:00:51 +0200 Subject: [PATCH 02/47] Use SimulatorIO as the "I/O driver" in the executor Entails making it Send + Sync, which may or may not be what we want. --- crates/runtime-core/src/sim/executor/mod.rs | 26 +++++- crates/runtime-core/src/sim/io/fs.rs | 50 +++++------ crates/runtime-core/src/sim/io/mod.rs | 92 ++++++++++----------- crates/runtime-core/src/sim/io/op.rs | 41 +++++---- 4 files changed, 114 insertions(+), 95 deletions(-) diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index fbb7f7c0cf2..eee88b4a88a 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -10,6 +10,8 @@ use core::{ use spin::Mutex; +use crate::sim::io::SimulatorIO; + use super::{time::TimeHandle, Rng}; mod task; @@ -21,11 +23,12 @@ type Runnable = async_task::Runnable; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RuntimeConfig { pub seed: u64, + pub enable_io: bool, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed } + Self { seed, enable_io: false } } } @@ -145,6 +148,12 @@ impl Runtime { } } + // TODO: This is a stopgap to allow submission of I/O tasks. We probably + // want the user-facing API to hide this. + pub fn io(&self) -> &Option { + &self.executor.io + } + /// Drive a top-level future to completion on the simulation executor. /// /// While the future runs, spawned tasks share the same deterministic @@ -360,6 +369,7 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, + io: Option, } impl Executor { @@ -375,6 +385,7 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), + io: config.enable_io.then(SimulatorIO::default), } } @@ -499,6 +510,10 @@ impl Executor { }; } + if self.run_pending_io() { + continue; + } + if self.time.wake_next_timer() { continue; } @@ -527,6 +542,15 @@ impl Executor { } } + fn run_pending_io(&self) -> bool { + // TODO: Inject faults (reorder, delay, drop, ..) when buggify is enabled. + // Also, should this run more than one queue entry? + match &self.io { + Some(io) => io.tick(), + None => false, + } + } + /// Look up the record for a node, panicking if the node is unknown. fn node_record(&self, node: NodeId) -> Arc { self.nodes diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 9c540f5e1b3..bdc64b87657 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,7 +1,7 @@ -use alloc::{collections::BTreeMap, rc::Rc}; +use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cell::{Cell, RefCell}, cmp, + sync::atomic::{AtomicU64, Ordering}, }; pub const PAGE_SIZE: usize = 4096; @@ -27,13 +27,13 @@ impl PageIndex { } struct Page { - bytes: RefCell<[u8; PAGE_SIZE]>, + bytes: spin::Mutex<[u8; PAGE_SIZE]>, } impl Page { fn zeroed() -> Self { Self { - bytes: RefCell::new([0; PAGE_SIZE]), + bytes: spin::Mutex::new([0; PAGE_SIZE]), } } } @@ -47,25 +47,25 @@ impl Page { /// or written. Writing a page is atomic. #[derive(Clone)] pub struct File { - pages: RefCell>>, - len: Cell, + pages: Arc>>>, + len: Arc, } impl File { - pub(super) const fn new() -> Self { + pub(super) fn new() -> Self { Self { - pages: RefCell::new(BTreeMap::new()), - len: Cell::new(0), + pages: Arc::new(spin::Mutex::new(BTreeMap::new())), + len: Arc::new(AtomicU64::new(0)), } } - pub(super) const fn len(&self) -> u64 { - self.len.get() + pub(super) fn len(&self) -> u64 { + self.len.load(Ordering::Relaxed) } #[allow(unused)] - pub(super) const fn is_empty(&self) -> bool { - self.len.get() == 0 + pub(super) fn is_empty(&self) -> bool { + self.len() == 0 } /// Change the file length. @@ -80,7 +80,7 @@ impl File { if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } - let old_len = self.len.get(); + let old_len = self.len(); match new_len.cmp(&old_len) { Equal => {} @@ -92,13 +92,13 @@ impl File { self.get_or_allocate_page(PageIndex(index)); } - self.len.set(new_len); + self.len.store(new_len, Ordering::Relaxed); } Less => { - self.len.set(new_len); + self.len.store(new_len, Ordering::Relaxed); let first_removed = PageIndex::from_offset(new_len); - let removed = self.pages.borrow_mut().split_off(&first_removed); + let removed = self.pages.lock().split_off(&first_removed); drop(removed); } } @@ -114,7 +114,7 @@ impl File { match self.get_page(PageIndex(index)) { Some(page) => { - dst.copy_from_slice(&*page.bytes.borrow()); + dst.copy_from_slice(&*page.bytes.lock()); } None => { dst.fill(0); @@ -131,24 +131,24 @@ impl File { } let page = self.get_or_allocate_page(PageIndex(index)); - page.bytes.borrow_mut().copy_from_slice(src); + page.bytes.lock().copy_from_slice(src); let end = index .checked_add(1) .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) .ok_or(Error::OffsetOverflow)?; - self.len.set(cmp::max(self.len.get(), end)); + self.len.fetch_max(end, Ordering::Relaxed); Ok(()) } - fn get_page(&self, index: PageIndex) -> Option> { - self.pages.borrow().get(&index).cloned() + fn get_page(&self, index: PageIndex) -> Option> { + self.pages.lock().get(&index).cloned() } - fn get_or_allocate_page(&self, index: PageIndex) -> Rc { - let mut pages = self.pages.borrow_mut(); - Rc::clone(pages.entry(index).or_insert_with(|| Rc::new(Page::zeroed()))) + fn get_or_allocate_page(&self, index: PageIndex) -> Arc { + let mut pages = self.pages.lock(); + Arc::clone(pages.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) } } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 670b68d7aa3..6f8f9af8569 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -1,15 +1,9 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, - rc::Rc, -}; -use core::{ - cell::RefCell, - future::{poll_fn, Future}, - pin::Pin, - result::Result, - task::Poll, + sync::Arc, }; +use core::result::Result; use futures_channel::oneshot; use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; @@ -35,42 +29,23 @@ impl From for Error { } } -#[derive(Default)] +#[derive(Clone, Default)] pub struct SimulatorIO { - inner: Rc>, + inner: Arc>, } impl SimulatorIO { - pub fn tick(&self) { - self.inner.borrow_mut().tick(); - } - - fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> oneshot::Receiver { - let (tx, rx) = oneshot::channel(); - self.inner.borrow_mut().submit(op(tx)); - rx - } - - // TODO: The sim runtime should be advancing I/O. Until it does, `tick()` - // whenever a result future is polled and returns pending. - async fn wait_for(&self, mut rx: oneshot::Receiver) -> Result { - poll_fn(|cx| match Pin::new(&mut rx).poll(cx) { - Poll::Ready(result) => Poll::Ready(result), - Poll::Pending => { - self.tick(); - cx.waker().wake_by_ref(); - Poll::Pending - } - }) - .await + pub fn tick(&self) -> bool { + self.inner.lock().tick() } async fn submit_and_wait( &self, op: impl FnOnce(oneshot::Sender) -> Box, ) -> Result { - let rx = self.submit(op); - self.wait_for(rx).await + let (tx, rx) = oneshot::channel(); + self.inner.lock().submit(op(tx)); + rx.await } } @@ -90,7 +65,7 @@ impl SpacetimeIO for SimulatorIO { .expect("`create_file` future cancelled") } - async fn write_all_at( + async fn write_all_at( &self, fd: Self::Fd, buf: B, @@ -113,13 +88,13 @@ impl SpacetimeIO for SimulatorIO { } else { let (tx, rx) = oneshot::channel(); for op in op::write_at(fd, buf, offset, tx) { - self.inner.borrow_mut().submit(op); + self.inner.lock().submit(op); } - self.wait_for(rx).await.expect("`write_all_at` future cancelled") + rx.await.expect("`write_all_at` future cancelled") } } - async fn read_exact_at( + async fn read_exact_at( &self, fd: Self::Fd, buf: B, @@ -142,9 +117,9 @@ impl SpacetimeIO for SimulatorIO { } else { let (tx, rx) = oneshot::channel(); for op in op::read_at(fd, buf, offset, tx) { - self.inner.borrow_mut().submit(op); + self.inner.lock().submit(op); } - self.wait_for(rx).await.expect("`read_exact_at` future cancelled") + rx.await.expect("`read_exact_at` future cancelled") } } @@ -175,13 +150,28 @@ struct SimulatorIOInner { } impl SimulatorIOInner { - fn tick(&mut self) { + // TODO: Allow runtime to inject faults via: + // + // - pick random entries from the submission queue + // - drop queue entries + // - delay `execute` (somehow) + // - delay `complete` + // - make a submission fail without performing its effect + // - execute an arbitrary number of (random) SQEs + // - complete an arbitrary number of CQEs + + fn tick(&mut self) -> bool { + let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { sqe.execute(&mut self.files, &mut self.completions); + progress = true; } if let Some(cqe) = self.completions.pop_front() { cqe.complete(); + progress = true; } + + progress } fn submit(&mut self, op: Box) { @@ -189,7 +179,7 @@ impl SimulatorIOInner { } } -trait Submission { +trait Submission: Send { fn execute( self: Box, files: &mut BTreeMap, fs::File>, @@ -197,20 +187,23 @@ trait Submission { ); } -trait Completion { +trait Completion: Send { fn complete(self: Box); } #[cfg(test)] mod tests { - use crate::sim::Runtime; + use crate::sim::{Runtime, RuntimeConfig}; use super::*; #[test] fn create_file() { - let mut rt = Runtime::new(1); - let io = SimulatorIO::default(); + let mut rt = Runtime::with_config(RuntimeConfig { + enable_io: true, + ..<_>::default() + }); + let io = rt.io().clone().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) @@ -240,8 +233,11 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::new(1); - let io = SimulatorIO::default(); + let mut rt = Runtime::with_config(RuntimeConfig { + enable_io: true, + ..<_>::default() + }); + let io = rt.io().clone().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 921129d2536..963f26dc522 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,9 +1,8 @@ use alloc::{ boxed::Box, collections::{btree_map, BTreeMap, VecDeque}, - rc::Rc, + sync::Arc, }; -use core::cell::RefCell; use futures_channel::oneshot; use super::{fs, Completion, Error, Submission}; @@ -12,7 +11,7 @@ use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; pub type WriteAtResult = Result>; pub type ReadAtResult = Result>; -pub fn write_at( +pub fn write_at( fd: fs::File, buf: B, offset: u64, @@ -21,7 +20,7 @@ pub fn write_at( let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; - let state = Rc::new(RefCell::new(PagedOpState { + let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), notify: Some(notify), remaining: page_count, @@ -40,7 +39,7 @@ pub fn write_at( }) } -pub fn read_at( +pub fn read_at( fd: fs::File, buf: B, offset: u64, @@ -49,7 +48,7 @@ pub fn read_at( let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; - let state = Rc::new(RefCell::new(PagedOpState { + let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), notify: Some(notify), remaining: page_count, @@ -96,11 +95,11 @@ struct GenericCompletion { notify: oneshot::Sender, } -fn completion(result: T, notify: oneshot::Sender) -> Box { +fn completion(result: T, notify: oneshot::Sender) -> Box { Box::new(GenericCompletion { result, notify }) } -impl Completion for GenericCompletion { +impl Completion for GenericCompletion { fn complete(self: Box) { let Self { result, notify } = *self; let _ = notify.send(result); @@ -120,7 +119,7 @@ impl Submission for Ready { } } -pub fn ready(result: T, notify: oneshot::Sender) -> Box { +pub fn ready(result: T, notify: oneshot::Sender) -> Box { Box::new(Ready(completion(result, notify))) } @@ -208,13 +207,13 @@ struct PagedOpState { first_error: Option, } -fn complete_page_op( - state: &Rc>>, +fn complete_page_op( + state: &Arc>>, result: Result<(), fs::Error>, completions: &mut VecDeque>, ) { let complete = { - let mut state = state.borrow_mut(); + let mut state = state.lock(); if let Err(e) = result && state.first_error.is_none() { @@ -232,13 +231,13 @@ fn complete_page_op( } struct WriteCompletion { - state: Rc>>, + state: Arc>>, } -impl Completion for WriteCompletion { +impl Completion for WriteCompletion { fn complete(self: Box) { let (notify, result) = { - let mut state = self.state.borrow_mut(); + let mut state = self.state.lock(); assert_eq!(state.remaining, 0); @@ -261,10 +260,10 @@ struct WritePage { fd: fs::File, file_page: usize, buf_page: usize, - state: Rc>>, + state: Arc>>, } -impl Submission for WritePage { +impl Submission for WritePage { fn execute( self: Box, _files: &mut BTreeMap, fs::File>, @@ -278,7 +277,7 @@ impl Submission for WritePage { } = *self; let result = { - let state_ref = state.borrow(); + let state_ref = state.lock(); let buf = state_ref.buf.as_ref().expect("buffer went away"); let start = buf_page * SECTOR_SIZE; @@ -293,10 +292,10 @@ struct ReadPage { fd: fs::File, file_page: usize, buf_page: usize, - state: Rc>>, + state: Arc>>, } -impl Submission for ReadPage { +impl Submission for ReadPage { fn execute( self: Box, _files: &mut BTreeMap, fs::File>, @@ -310,7 +309,7 @@ impl Submission for ReadPage { } = *self; let result = { - let mut state_ref = state.borrow_mut(); + let mut state_ref = state.lock(); let buf = state_ref.buf.as_mut().expect("buffer went away"); let start = buf_page * SECTOR_SIZE; From f47352a360197dd259fb13c7903b502885c05a7b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 4 Aug 2026 16:29:02 +0200 Subject: [PATCH 03/47] Make it clearer that we're clearing --- crates/runtime-core/src/sim/io/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 6f8f9af8569..91ac80527b4 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -214,6 +214,12 @@ mod tests { #[repr(C, align(4096))] struct Buf([u8; 2 * SECTOR_SIZE]); + impl Buf { + fn clear(&mut self) { + self.0.fill(0); + } + } + impl AlignedBytes for Buf { fn as_bytes(&self) -> &[u8] { &self.0 @@ -246,7 +252,7 @@ mod tests { .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) .map_err(|ErrorWith { error, .. }| error) .unwrap(); - buf.0.fill(0); + buf.clear(); let buf = rt .block_on(io.read_exact_at(fd, buf, 0)) .map_err(|ErrorWith { error, .. }| error) From 243930b95fb2503e7852c7a4f457cb0d787f5ecb Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 6 Aug 2026 18:27:58 +0200 Subject: [PATCH 04/47] Expose ways for the runtime to inject failures. --- crates/runtime-core/src/sim/io/mod.rs | 78 +++++++-- crates/runtime-core/src/sim/io/op.rs | 234 ++++++++++++++++---------- 2 files changed, 205 insertions(+), 107 deletions(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 91ac80527b4..e50c77f558a 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -2,14 +2,19 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, sync::Arc, + vec::Vec, }; -use core::result::Result; +use core::{ops::RangeBounds, result::Result}; use futures_channel::oneshot; -use crate::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use crate::{ + io::{AlignedBytes, ErrorWith, SpacetimeIO}, + sim::Rng, +}; mod fs; -mod op; +pub mod op; +use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; @@ -31,14 +36,46 @@ impl From for Error { #[derive(Clone, Default)] pub struct SimulatorIO { + // TODO: We make `SimulatorIO` `Send + Sync` for now, because + // [crate::sim::executor::Handle] is just `Arc`. This means that a + // future carrying a handle can't be `spawn`ed, because spawning requires + // the future to be `Send`. + // + // We should fix this at some point, so below can become `Rc>`. inner: Arc>, } impl SimulatorIO { + /// Run the submission at the front of the queue (if any), and complete the + /// completion at the front of the queue (if any). pub fn tick(&self) -> bool { self.inner.lock().tick() } + /// Execute `sqe`. + pub fn execute(&self, sqe: Box) { + self.inner.lock().execute(sqe); + } + + /// Remove and return the submission at the fron of the queue, if any. + pub fn next_submission(&self) -> Option> { + self.inner.lock().next() + } + + /// Remove and return a random submission, or `None` if the queue is empty. + pub fn random_submission(&self, rng: &Rng) -> Option> { + self.inner.lock().next_random(rng) + } + + /// Remove `range` from the completion queue. + pub fn completions(&self, range: impl RangeBounds) -> impl Iterator> { + self.inner + .lock() + .drain_completions(range) + .collect::>() + .into_iter() + } + async fn submit_and_wait( &self, op: impl FnOnce(oneshot::Sender) -> Box, @@ -163,7 +200,9 @@ impl SimulatorIOInner { fn tick(&mut self) -> bool { let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { - sqe.execute(&mut self.files, &mut self.completions); + if let Some(cqe) = sqe.execute(&mut self.files) { + self.completions.push_back(cqe); + } progress = true; } if let Some(cqe) = self.completions.pop_front() { @@ -174,21 +213,28 @@ impl SimulatorIOInner { progress } - fn submit(&mut self, op: Box) { - self.submissions.push_back(op); + fn execute(&mut self, sqe: Box) { + if let Some(cqe) = sqe.execute(&mut self.files) { + self.completions.push_back(cqe); + } } -} -trait Submission: Send { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ); -} + fn next(&mut self) -> Option> { + self.submissions.pop_front() + } + + fn next_random(&mut self, rng: &Rng) -> Option> { + let i = rng.next_u64() % self.submissions.len() as u64; + self.submissions.remove(i as usize) + } + + fn drain_completions(&mut self, range: impl RangeBounds) -> impl Iterator> { + self.completions.drain(range) + } -trait Completion: Send { - fn complete(self: Box); + fn submit(&mut self, op: Box) { + self.submissions.push_back(op); + } } #[cfg(test)] diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 963f26dc522..a4e95d71fb6 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,28 +1,60 @@ +use core::any::Any; + use alloc::{ boxed::Box, - collections::{btree_map, BTreeMap, VecDeque}, + collections::{btree_map, BTreeMap}, sync::Arc, }; use futures_channel::oneshot; -use super::{fs, Completion, Error, Submission}; +use super::{fs, Error}; use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; +/// An operation that can be submitted to the [super::SimulatorIO] driver. +pub trait Submission: Send + Any { + /// Run the operations with mutable access to the currently registered + /// [fs::File]s. + /// + /// If the operation is done, a [Completion] is returned in a `Some`. + /// `None` may be returned if: + /// + /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. + /// - The submission is a [Noop]. + /// + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; +} + +/// An object containing the result of executing a [Submission], as well as a +/// handle to resolve a future waiting on the outcome of the operation. +pub trait Completion: Send { + /// Resolve the future waiting on the outcome of the operation. + fn complete(self: Box); +} + +/// A channel to resolve a future waiting on the outcome of a submitted +/// operation. +pub type OnComplete = oneshot::Sender; + pub type WriteAtResult = Result>; -pub type ReadAtResult = Result>; +/// Write the contents of `buf` to `fd` at `offset`. +/// +/// This operation is split into multiple writes to individual pages. The +/// `on_complete` future resolves only after all page writes completed. +/// +/// Ownership of `buf` is transferred back when the operation completes. pub fn write_at( fd: fs::File, buf: B, offset: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, ) -> impl Iterator> { let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), - notify: Some(notify), + on_complete: Some(on_complete), remaining: page_count, first_error: None, })); @@ -39,18 +71,27 @@ pub fn write_at( }) } +pub type ReadAtResult = Result>; + +/// Fill `buf` by reading from `fd` at `offset`. +/// +/// This operation is split into multple reads from the individual pages needed +/// to fill `buf`. The `on_complete` future resolves only after all page reads +/// completed. +/// +/// Ownership of `buf` is transferred back when the operation completes. pub fn read_at( fd: fs::File, buf: B, offset: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, ) -> impl Iterator> { let first_page = (offset / SECTOR_SIZE as u64) as usize; let page_count = buf.as_bytes().len() / SECTOR_SIZE; let state = Arc::new(spin::Mutex::new(PagedOpState { buf: Some(buf), - notify: Some(notify), + on_complete: Some(on_complete), remaining: page_count, first_error: None, })); @@ -67,92 +108,107 @@ pub fn read_at( }) } -pub fn open_file(path: &str, notify: oneshot::Sender>) -> Box { +/// Open file at `path`. +pub fn open_file(path: &str, on_complete: OnComplete>) -> Box { Box::new(OpenFile { path: path.into(), - notify, + on_complete, }) } -pub fn create_file(path: &str, len: u64, notify: oneshot::Sender>) -> Box { +/// Create a new file at `path` and allocate `len` space for it. +pub fn create_file(path: &str, len: u64, on_complete: OnComplete>) -> Box { Box::new(CreateFile { path: path.into(), len, - notify, + on_complete, }) } -pub fn get_len(fd: fs::File, notify: oneshot::Sender>) -> Box { - Box::new(GetLen { fd, notify }) +/// Get the length of the file `fd`. +pub fn get_len(fd: fs::File, on_complete: OnComplete>) -> Box { + Box::new(GetLen { fd, on_complete }) } -pub fn set_len(fd: fs::File, len: u64, notify: oneshot::Sender>) -> Box { - Box::new(SetLen { fd, len, notify }) +/// Set the length of the file `fd`. +pub fn set_len(fd: fs::File, len: u64, on_complete: OnComplete>) -> Box { + Box::new(SetLen { fd, len, on_complete }) } struct GenericCompletion { result: T, - notify: oneshot::Sender, + on_complete: OnComplete, } -fn completion(result: T, notify: oneshot::Sender) -> Box { - Box::new(GenericCompletion { result, notify }) +fn completion(result: T, on_complete: OnComplete) -> Box { + Box::new(GenericCompletion { result, on_complete }) } impl Completion for GenericCompletion { fn complete(self: Box) { - let Self { result, notify } = *self; - let _ = notify.send(result); + let Self { + result, on_complete, .. + } = *self; + let _ = on_complete.send(result); } } -struct Ready(Box); +/// [Submission] created by [noop]. +pub(crate) struct Noop; + +impl Submission for Noop { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + None + } +} + +/// An operation that does nothing. +/// +/// Note that no completion is associated with a noop, but the submission still +/// occupies a slot in the submission queue. +pub fn noop() -> Box { + Box::new(Noop) +} + +/// [Submission] created by [ready]. +pub(crate) struct Ready(Box); impl Submission for Ready { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self(completion) = *self; - completions.push_back(completion); + Some(completion) } } -pub fn ready(result: T, notify: oneshot::Sender) -> Box { - Box::new(Ready(completion(result, notify))) +/// An operation that is already complete with `result`. +pub fn ready(result: T, on_complete: OnComplete) -> Box { + Box::new(Ready(completion(result, on_complete))) } -struct OpenFile { +/// [Submission] created by [open_file]. +pub(crate) struct OpenFile { path: Box, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for OpenFile { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { path, notify } = *self; + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { path, on_complete } = *self; let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct CreateFile { +/// [Submission] created by [create_file]. +pub(crate) struct CreateFile { path: Box, len: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for CreateFile { - fn execute( - self: Box, - files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { path, len, notify } = *self; + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { path, len, on_complete } = *self; let result = (|| { let file = match files.entry(path.clone()) { btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), @@ -161,48 +217,42 @@ impl Submission for CreateFile { file.set_len(len)?; Ok(file) })(); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct GetLen { +/// [Submission] created by [get_len]. +pub(crate) struct GetLen { fd: fs::File, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for GetLen { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { fd, notify } = *self; + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + let Self { fd, on_complete } = *self; let result = Ok(fd.len()); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } -struct SetLen { +/// [Submission] created by [set_len]. +pub(crate) struct SetLen { fd: fs::File, len: u64, - notify: oneshot::Sender>, + on_complete: OnComplete>, } impl Submission for SetLen { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { - let Self { fd, len, notify } = *self; + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { + let Self { fd, len, on_complete } = *self; let result = fd.set_len(len).map_err(Error::from); - completions.push_back(completion(result, notify)); + Some(completion(result, on_complete)) } } struct PagedOpState { buf: Option, - notify: Option>>>, + on_complete: Option>>>, remaining: usize, first_error: Option, } @@ -210,8 +260,7 @@ struct PagedOpState { fn complete_page_op( state: &Arc>>, result: Result<(), fs::Error>, - completions: &mut VecDeque>, -) { +) -> Option>> { let complete = { let mut state = state.lock(); if let Err(e) = result @@ -225,38 +274,36 @@ fn complete_page_op( state.remaining == 0 }; - if complete { - completions.push_back(Box::new(WriteCompletion { state: state.clone() })); - } + complete.then(|| Box::new(PageOpCompletion { state: state.clone() })) } -struct WriteCompletion { +struct PageOpCompletion { state: Arc>>, } -impl Completion for WriteCompletion { +impl Completion for PageOpCompletion { fn complete(self: Box) { - let (notify, result) = { + let (on_complete, result) = { let mut state = self.state.lock(); assert_eq!(state.remaining, 0); let buf = state.buf.take().expect("write completed more than once"); - let notify = state.notify.take().expect("write completed more than once"); + let on_complete = state.on_complete.take().expect("write completed more than once"); let result = match state.first_error.take() { None => Ok(buf), Some(error) => Err(ErrorWith { error, with: buf }), }; - (notify, result) + (on_complete, result) }; - let _ = notify.send(result); + let _ = on_complete.send(result); } } -struct WritePage { +pub(crate) struct WritePage { fd: fs::File, file_page: usize, buf_page: usize, @@ -264,11 +311,7 @@ struct WritePage { } impl Submission for WritePage { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, file_page, @@ -284,11 +327,11 @@ impl Submission for WritePage { let end = start + SECTOR_SIZE; fd.write_page(&buf.as_bytes()[start..end], file_page as _) }; - complete_page_op(&state, result, completions); + complete_page_op(&state, result).map(|c| c as Box) } } -struct ReadPage { +pub(crate) struct ReadPage { fd: fs::File, file_page: usize, buf_page: usize, @@ -296,11 +339,7 @@ struct ReadPage { } impl Submission for ReadPage { - fn execute( - self: Box, - _files: &mut BTreeMap, fs::File>, - completions: &mut VecDeque>, - ) { + fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, file_page, @@ -316,6 +355,19 @@ impl Submission for ReadPage { let end = start + SECTOR_SIZE; fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) }; - complete_page_op(&state, result, completions); + complete_page_op(&state, result).map(|c| c as Box) + } +} + +#[cfg(test)] +mod tests { + use core::any::Any; + + use super::*; + + #[test] + fn downcast() { + let sqe: Box = noop(); + sqe.downcast::().unwrap(); } } From 40faf30984511ed7963b5d3a66963390ae5c490b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 14:01:11 +0200 Subject: [PATCH 05/47] Encapsulate SimulatorIO in an I/O "driver" that can inject failures. --- crates/runtime-core/src/sim/executor/io.rs | 101 ++++++++++++++++++++ crates/runtime-core/src/sim/executor/mod.rs | 45 +++++---- crates/runtime-core/src/sim/io/fs.rs | 6 +- crates/runtime-core/src/sim/io/mod.rs | 92 +++++++++++------- crates/runtime-core/src/sim/io/op.rs | 75 ++++++++++++++- 5 files changed, 262 insertions(+), 57 deletions(-) create mode 100644 crates/runtime-core/src/sim/executor/io.rs diff --git a/crates/runtime-core/src/sim/executor/io.rs b/crates/runtime-core/src/sim/executor/io.rs new file mode 100644 index 00000000000..ca1831978bf --- /dev/null +++ b/crates/runtime-core/src/sim/executor/io.rs @@ -0,0 +1,101 @@ +use crate::sim::{io::SimulatorIO, Rng}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Config { + /// The max number of submissions to run per [Driver::tick]. + pub max_submissions_per_tick: usize, + /// The max number of completions to finish per [Driver::tick]. + pub max_completions_per_tick: usize, + /// Submission reordering probability. + /// + /// Describes the probability by which to select the next submission queue + /// entry randomly, as opposed to the oldest entry in the queue. + pub prob_reorder_submissions: f64, + /// Completion reordering probability. + /// + /// Describes the probability by which to select the next completion queue + /// entry randomly, as opposed to the oldest entry in the queue. + pub prob_reorder_completions: f64, + /// Probability by which to skip one submission queue entry. + /// + /// If skipped, the entry still counts towards `max_submissions_per_tick`. + pub prob_skip: f64, + /// Probability by which to cancel a submission queue entry. + /// + /// [crate::sim::io::op::Submission::cancel()] is called on the entry, which + /// may generate a completion. + pub prob_cancel: f64, +} + +impl Default for Config { + fn default() -> Self { + Self { + max_submissions_per_tick: 1, + max_completions_per_tick: 1, + prob_reorder_submissions: 0.0, + prob_reorder_completions: 0.0, + prob_skip: 0.0, + prob_cancel: 0.0, + } + } +} + +pub struct Driver { + io: SimulatorIO, + config: Config, +} + +impl Driver { + pub fn new(config: Config) -> Self { + Self { + io: <_>::default(), + config, + } + } + + /// Advance the I/O simulator according the [Config]. + /// + /// Returns `true` if progress has been made, or there are pending entries + /// in either the submission or completion queue. + pub fn tick(&self, rng: &Rng) -> bool { + let mut progress = false; + for _ in 0..self.config.max_submissions_per_tick { + if !rng.buggify_with_prob(self.config.prob_skip) { + let sqe = if rng.buggify_with_prob(self.config.prob_reorder_submissions) { + self.io.random_submission(rng) + } else { + self.io.next_submission() + }; + + if let Some(sqe) = sqe { + if rng.buggify_with_prob(self.config.prob_cancel) { + sqe.cancel(); + } else { + self.io.execute(sqe); + } + progress = true; + } + } + } + + for _ in 0..self.config.max_completions_per_tick { + let cqe = if rng.buggify_with_prob(self.config.prob_reorder_completions) { + self.io.random_completion(rng) + } else { + self.io.next_completion() + }; + + if let Some(cqe) = cqe { + cqe.complete(); + progress = true + } + } + + progress |= self.io.pending(); + progress + } + + pub fn io(&self) -> &SimulatorIO { + &self.io + } +} diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index eee88b4a88a..1913329b2e2 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -14,21 +14,34 @@ use crate::sim::io::SimulatorIO; use super::{time::TimeHandle, Rng}; +mod io; + mod task; use task::Abortable; pub use task::{AbortHandle, JoinError, JoinHandle}; type Runnable = async_task::Runnable; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] pub struct RuntimeConfig { pub seed: u64, - pub enable_io: bool, + pub io: Option, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed, enable_io: false } + Self { seed, io: None } + } + + pub fn enable_io(self) -> Self { + Self { + io: Some(self.io.unwrap_or_default()), + ..self + } + } + + pub fn with_io_config(self, io: Option) -> Self { + Self { io, ..self } } } @@ -150,8 +163,8 @@ impl Runtime { // TODO: This is a stopgap to allow submission of I/O tasks. We probably // want the user-facing API to hide this. - pub fn io(&self) -> &Option { - &self.executor.io + pub fn io(&self) -> Option<&SimulatorIO> { + self.executor.io.as_ref().map(|driver| driver.io()) } /// Drive a top-level future to completion on the simulation executor. @@ -369,7 +382,7 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, - io: Option, + io: Option, } impl Executor { @@ -385,7 +398,7 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), - io: config.enable_io.then(SimulatorIO::default), + io: config.io.map(io::Driver::new), } } @@ -502,6 +515,7 @@ impl Executor { loop { self.run_all_ready(); + let pending_io = self.drive_io(); if task.is_finished() { let waker = Waker::noop(); return match Pin::new(&mut task).poll(&mut Context::from_waker(waker)) { @@ -510,11 +524,7 @@ impl Executor { }; } - if self.run_pending_io() { - continue; - } - - if self.time.wake_next_timer() { + if self.time.wake_next_timer() || pending_io { continue; } @@ -542,12 +552,11 @@ impl Executor { } } - fn run_pending_io(&self) -> bool { - // TODO: Inject faults (reorder, delay, drop, ..) when buggify is enabled. - // Also, should this run more than one queue entry? - match &self.io { - Some(io) => io.tick(), - None => false, + fn drive_io(&self) -> bool { + if let Some(io) = &self.io { + io.tick(&self.rng) + } else { + false } } diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index bdc64b87657..908c8b7ba83 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -74,7 +74,7 @@ impl File { /// /// Extending allocates pages eagerly as needed. Shrinking drops all pages /// at or beyond the new EOF. - pub fn set_len(&self, new_len: u64) -> Result<()> { + pub(super) fn set_len(&self, new_len: u64) -> Result<()> { use cmp::Ordering::*; if !new_len.is_multiple_of(PAGE_SIZE_U64) { @@ -107,7 +107,7 @@ impl File { } /// Read one complete page. - pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { + pub(super) fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { if dst.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } @@ -125,7 +125,7 @@ impl File { } /// Write one complete page. - pub fn write_page(&self, src: &[u8], index: u64) -> Result<()> { + pub(super) fn write_page(&self, src: &[u8], index: u64) -> Result<()> { if src.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index e50c77f558a..c9a69fcb8ba 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -2,9 +2,8 @@ use alloc::{ boxed::Box, collections::{BTreeMap, VecDeque}, sync::Arc, - vec::Vec, }; -use core::{ops::RangeBounds, result::Result}; +use core::{num::NonZeroUsize, result::Result}; use futures_channel::oneshot; use crate::{ @@ -21,11 +20,23 @@ pub use fs::File; #[derive(Debug)] pub enum Error { - FileNotFound { path: Box }, - FileAlreadyExists { path: Box }, - ShortWrite { expected: usize, written: usize }, - UnexpectedEof { expected: usize, read: usize }, + FileNotFound { + path: Box, + }, + FileAlreadyExists { + path: Box, + }, + ShortWrite { + expected: usize, + written: usize, + }, + UnexpectedEof { + expected: usize, + read: usize, + }, Fs(fs::Error), + /// Injected by the I/O driver. + Cancelled, } impl From for Error { @@ -46,6 +57,23 @@ pub struct SimulatorIO { } impl SimulatorIO { + /// Returns `true` if there are entries in either the submission or + /// completion queues. + pub fn pending(&self) -> bool { + let inner = self.inner.lock(); + inner.submissions.len() + inner.completions.len() > 0 + } + + /// Number of entries in the submission queue. + pub fn pending_submissions(&self) -> usize { + self.inner.lock().submissions.len() + } + + /// Number of entries in the completion queue. + pub fn pending_completions(&self) -> usize { + self.inner.lock().completions.len() + } + /// Run the submission at the front of the queue (if any), and complete the /// completion at the front of the queue (if any). pub fn tick(&self) -> bool { @@ -57,23 +85,24 @@ impl SimulatorIO { self.inner.lock().execute(sqe); } - /// Remove and return the submission at the fron of the queue, if any. + /// Remove and return the submission at the front of the queue, if any. pub fn next_submission(&self) -> Option> { - self.inner.lock().next() + self.inner.lock().next_submission() } /// Remove and return a random submission, or `None` if the queue is empty. pub fn random_submission(&self, rng: &Rng) -> Option> { - self.inner.lock().next_random(rng) + self.inner.lock().random_submission(rng) + } + + /// Remove and return the completion at the front of the queue, if any. + pub fn next_completion(&self) -> Option> { + self.inner.lock().next_completion() } - /// Remove `range` from the completion queue. - pub fn completions(&self, range: impl RangeBounds) -> impl Iterator> { - self.inner - .lock() - .drain_completions(range) - .collect::>() - .into_iter() + /// Remove and return a random completion, or `None` if the queue is empty. + pub fn random_completion(&self, rng: &Rng) -> Option> { + self.inner.lock().random_completion(rng) } async fn submit_and_wait( @@ -219,17 +248,22 @@ impl SimulatorIOInner { } } - fn next(&mut self) -> Option> { + fn next_submission(&mut self) -> Option> { self.submissions.pop_front() } - fn next_random(&mut self, rng: &Rng) -> Option> { - let i = rng.next_u64() % self.submissions.len() as u64; - self.submissions.remove(i as usize) + fn random_submission(&mut self, rng: &Rng) -> Option> { + let len = NonZeroUsize::new(self.submissions.len())?; + self.submissions.remove(rng.index(len.get())) + } + + fn next_completion(&mut self) -> Option> { + self.completions.pop_front() } - fn drain_completions(&mut self, range: impl RangeBounds) -> impl Iterator> { - self.completions.drain(range) + fn random_completion(&mut self, rng: &Rng) -> Option> { + let len = NonZeroUsize::new(self.completions.len())?; + self.completions.remove(rng.index(len.get())) } fn submit(&mut self, op: Box) { @@ -245,11 +279,8 @@ mod tests { #[test] fn create_file() { - let mut rt = Runtime::with_config(RuntimeConfig { - enable_io: true, - ..<_>::default() - }); - let io = rt.io().clone().unwrap(); + let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); + let io = rt.io().cloned().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) @@ -285,11 +316,8 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::with_config(RuntimeConfig { - enable_io: true, - ..<_>::default() - }); - let io = rt.io().clone().unwrap(); + let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); + let io = rt.io().cloned().unwrap(); let fd = rt .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index a4e95d71fb6..3c2b1f90086 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -22,6 +22,16 @@ pub trait Submission: Send + Any { /// - The submission is a [Noop]. /// fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; + + /// Cancel the operation instead of executing it. + /// + /// This will generate a [Completion] with the result [Error::Cancelled], + /// unless: + /// + /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. + /// - The submission is a [Noop]. + /// + fn cancel(self: Box) -> Option>; } /// An object containing the result of executing a [Submission], as well as a @@ -160,6 +170,10 @@ impl Submission for Noop { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { None } + + fn cancel(self: Box) -> Option> { + None + } } /// An operation that does nothing. @@ -178,6 +192,11 @@ impl Submission for Ready { let Self(completion) = *self; Some(completion) } + + fn cancel(self: Box) -> Option> { + let Self(completion) = *self; + Some(completion) + } } /// An operation that is already complete with `result`. @@ -197,6 +216,11 @@ impl Submission for OpenFile { let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { path: _, on_complete } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [create_file]. @@ -219,6 +243,15 @@ impl Submission for CreateFile { })(); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { + path: _, + len: _, + on_complete, + } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [get_len]. @@ -233,6 +266,11 @@ impl Submission for GetLen { let result = Ok(fd.len()); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { fd: _, on_complete } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } /// [Submission] created by [set_len]. @@ -248,6 +286,15 @@ impl Submission for SetLen { let result = fd.set_len(len).map_err(Error::from); Some(completion(result, on_complete)) } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + len: _, + on_complete, + } = *self; + Some(completion(Err(Error::Cancelled), on_complete)) + } } struct PagedOpState { @@ -259,14 +306,14 @@ struct PagedOpState { fn complete_page_op( state: &Arc>>, - result: Result<(), fs::Error>, + result: Result<(), Error>, ) -> Option>> { let complete = { let mut state = state.lock(); if let Err(e) = result && state.first_error.is_none() { - state.first_error.replace(e.into()); + state.first_error.replace(e); } assert!(state.remaining > 0); state.remaining -= 1; @@ -327,7 +374,17 @@ impl Submission for WritePage { let end = start + SECTOR_SIZE; fd.write_page(&buf.as_bytes()[start..end], file_page as _) }; - complete_page_op(&state, result).map(|c| c as Box) + complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) + } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + file_page: _, + buf_page: _, + state, + } = *self; + complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) } } @@ -355,7 +412,17 @@ impl Submission for ReadPage { let end = start + SECTOR_SIZE; fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) }; - complete_page_op(&state, result).map(|c| c as Box) + complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) + } + + fn cancel(self: Box) -> Option> { + let Self { + fd: _, + file_page: _, + buf_page: _, + state, + } = *self; + complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) } } From c60c637ce2bc18f1a760f04a23d9f510e269fcec Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 14:08:19 +0200 Subject: [PATCH 06/47] Remove TODO --- crates/runtime-core/src/sim/io/mod.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index c9a69fcb8ba..6ea3a2eaa1d 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -216,16 +216,6 @@ struct SimulatorIOInner { } impl SimulatorIOInner { - // TODO: Allow runtime to inject faults via: - // - // - pick random entries from the submission queue - // - drop queue entries - // - delay `execute` (somehow) - // - delay `complete` - // - make a submission fail without performing its effect - // - execute an arbitrary number of (random) SQEs - // - complete an arbitrary number of CQEs - fn tick(&mut self) -> bool { let mut progress = false; if let Some(sqe) = self.submissions.pop_front() { From 51b1f6e51d8110b26a72849af27e8b192af39630 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 15:30:52 +0200 Subject: [PATCH 07/47] Fix optional dependencies --- crates/runtime/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 2b2cffc5317..d23741ce139 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -11,10 +11,12 @@ workspace = true [dependencies] tokio.workspace = true -spacetimedb-runtime-core = { workspace = true, optional = true } -libc = { version = "0.2", optional = true } +spacetimedb-runtime-core = { workspace = true } static_assertions = "1.1" +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } @@ -22,4 +24,4 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } futures.workspace = true [features] -simulation = ["dep:spacetimedb-runtime-core", "spacetimedb-runtime-core/sim", "dep:libc"] +simulation = ["spacetimedb-runtime-core/sim"] From 36ca1f48c7ed13003b2b8958044e4c05a979538c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 7 Aug 2026 17:02:48 +0200 Subject: [PATCH 08/47] Fix windows --- crates/runtime/src/io/tokio.rs | 74 ++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index eb948ed7e1c..7907a3f7cab 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -66,16 +66,9 @@ impl SpacetimeIO for TokioIO { offset: u64, ) -> Result> { let _rt = self.rt.enter(); - asyncify(move || { - #[cfg(unix)] - let res = fd.write_all_at(buf.as_bytes(), offset); - #[cfg(windows)] - let res = fd.seek_write(buf.as_bytes(), offset); - - match res { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - } + asyncify(move || match write_all_at(&fd, buf.as_bytes(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), }) .await } @@ -87,16 +80,9 @@ impl SpacetimeIO for TokioIO { offset: u64, ) -> Result> { let _rt = self.rt.enter(); - asyncify(move || { - #[cfg(unix)] - let res = fd.read_exact_at(buf.as_bytes_mut(), offset); - #[cfg(windows)] - let res = fd.seek_read(buf.as_bytes_mut(), offset); - - match res { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - } + asyncify(move || match read_exact_at(&fd, buf.as_bytes_mut(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), }) .await } @@ -154,10 +140,56 @@ async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result io::Result { options .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) .open(path) .await } + +#[cfg(unix)] +#[inline] +fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { + fd.read_exact_at(buf, offset) +} + +#[cfg(windows)] +fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match file.seek_read(buf, offset) { + Ok(0) => return Err(ErrorKind::UnexpectedEof.into()), + Ok(n) => { + offset += n as u64; + buf = &mut buf[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + + Ok(()) +} + +#[cfg(unix)] +#[inline] +fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { + fd.write_all_at(buf, offset) +} + +#[cfg(windows)] +fn write_all_at(fd: &std::fd::File, buf: &[u8], offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match file.seek_write(buf, offset) { + Ok(0) => return Err(ErrorKind::WriteZero.into()), + Ok(n) => { + offset += n as u64; + buf = &buf[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + + Ok(()) +} From 45362078c27de7f67d5b91fa9748f907aa03fc69 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Sat, 8 Aug 2026 12:24:23 +0200 Subject: [PATCH 09/47] Fix fix windows --- crates/runtime/src/io/tokio.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 7907a3f7cab..dcc77dbc5b4 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -141,7 +141,7 @@ async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result io::Result { +async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { options .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) .open(path) @@ -155,15 +155,15 @@ fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result< } #[cfg(windows)] -fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], mut offset: u64) -> io::Result<()> { +fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { while !buf.is_empty() { - match file.seek_read(buf, offset) { - Ok(0) => return Err(ErrorKind::UnexpectedEof.into()), + match fd.seek_read(buf, offset) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), Ok(n) => { offset += n as u64; buf = &mut buf[n..]; } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} Err(e) => return Err(e), } } @@ -178,15 +178,15 @@ fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { } #[cfg(windows)] -fn write_all_at(fd: &std::fd::File, buf: &[u8], offset: u64) -> io::Result<()> { +fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { while !buf.is_empty() { - match file.seek_write(buf, offset) { - Ok(0) => return Err(ErrorKind::WriteZero.into()), + match fd.seek_write(buf, offset) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), Ok(n) => { offset += n as u64; buf = &buf[n..]; } - Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} Err(e) => return Err(e), } } From d4da8686eb198815dea6b8bde875e8b5bf2fdb86 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Sat, 8 Aug 2026 13:35:46 +0200 Subject: [PATCH 10/47] Add a length function for files --- crates/runtime-core/src/io/mod.rs | 5 +++++ crates/runtime-core/src/sim/io/mod.rs | 9 +++++---- crates/runtime/src/io/tokio.rs | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index b4ff3744916..24a27579a2d 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -145,4 +145,9 @@ pub trait SpacetimeIO { /// Allocate `additional` bytes for the file `fd`. fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; + + /// Determine the length of the file `fd`. + /// + /// This should not depend on `fsync`, i.e. `statx`. See `std::io::Seek::stream_len`. + fn length(&self, fd: Self::Fd) -> Self::Completion>; } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 6ea3a2eaa1d..bb8ed855106 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -198,14 +198,15 @@ impl SpacetimeIO for SimulatorIO { } async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let len = self - .submit_and_wait(|tx| op::get_len(fd.clone(), tx)) - .await - .expect("`get_len` future cancelled")?; + let len = self.length(fd.clone()).await?; self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) .await .expect("`set_len` future cancelled") } + + fn length(&self, fd: Self::Fd) -> Self::Completion> { + self.submit(op::get_len(fd)) + } } #[derive(Default)] diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index dcc77dbc5b4..2dcf2ab1ab6 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -1,3 +1,8 @@ +use std::io::{Seek, SeekFrom}; +use std::panic; +use std::path::PathBuf; +use std::pin::Pin; +use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; #[cfg(unix)] @@ -107,6 +112,15 @@ impl SpacetimeIO for TokioIO { }) .await } + + fn length(&self, fd: Self::Fd) -> Self::Completion> { + self.rt + .spawn_blocking(move || { + let mut fd = fd.try_clone()?; + file_length(&mut fd) + }) + .into() + } } async fn asyncify(f: F) -> R From be161e4c19be1b28f1f3601e9a2364743d40d479 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 18 Aug 2026 16:36:07 +0200 Subject: [PATCH 11/47] Fix editor fuckup, satisfy error trait bound --- Cargo.lock | 1 + crates/runtime-core/Cargo.toml | 1 + crates/runtime-core/src/sim/io/fs.rs | 5 ++++- crates/runtime-core/src/sim/io/mod.rs | 26 +++++++++++--------------- crates/runtime/src/io/tokio.rs | 1 - 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0289ef17a31..c38b884a6ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8345,6 +8345,7 @@ dependencies = [ "async-task", "futures-channel", "spin", + "thiserror 2.0.17", "zerocopy", ] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 6ac037162dd..6b644b6ab92 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -17,4 +17,5 @@ sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] async-task = { version = "4.4", default-features = false, optional = true } futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } +thiserror = { version = "2.0", default-features = false } zerocopy = "0.8" diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 908c8b7ba83..efa3d9ab755 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -7,10 +7,13 @@ use core::{ pub const PAGE_SIZE: usize = 4096; const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] pub enum Error { + #[error("unaligned offset")] UnalignedOffset, + #[error("unaligned buffer")] UnalignedBuffer, + #[error("offset overflow")] OffsetOverflow, } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index bb8ed855106..c81cb5ca270 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -18,24 +18,20 @@ use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { - FileNotFound { - path: Box, - }, - FileAlreadyExists { - path: Box, - }, - ShortWrite { - expected: usize, - written: usize, - }, - UnexpectedEof { - expected: usize, - read: usize, - }, + #[error("file not found")] + FileNotFound { path: Box }, + #[error("file already exists")] + FileAlreadyExists { path: Box }, + #[error("failed to write expected number of bytes")] + ShortWrite { expected: usize, written: usize }, + #[error("unexpected eof")] + UnexpectedEof { expected: usize, read: usize }, + #[error(transparent)] Fs(fs::Error), /// Injected by the I/O driver. + #[error("operation cancelled")] Cancelled, } diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 2dcf2ab1ab6..d48e6623a62 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -17,7 +17,6 @@ use tokio::{runtime, task::spawn_blocking}; /// Implementation of [SpacetimeIO] that runs on a tokio runtime. pub struct TokioIO { - // TODO: Should this be [runtime::Runtime]? rt: runtime::Handle, // Ensure I/O stays on a single thread. _not_send: PhantomData>, From d498da1ae99a684ad4a5bee2ec3f1d05102ca0dd Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 21 Aug 2026 10:43:14 +0200 Subject: [PATCH 12/47] Unify completion future --- crates/runtime-core/src/io/mod.rs | 14 +- crates/runtime-core/src/sim/io/mod.rs | 141 +++++++------ crates/runtime-core/src/sim/io/op.rs | 265 +++++++++++++++++------- crates/runtime/src/io/tokio.rs | 280 +++++++++++++++----------- 4 files changed, 433 insertions(+), 267 deletions(-) diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index 24a27579a2d..7b54b1dd39b 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -101,12 +101,12 @@ pub trait SpacetimeIO { type Error; /// Open the file at `path`. - fn open_file(&self, path: &str) -> impl Future>; + fn open_file(&self, path: &str) -> Self::Completion>; /// Create the file at `path` and allocate `len` bytes. /// /// Returns an error if the file already exists. - fn create_file(&self, path: &str, len: u64) -> impl Future>; + fn create_file(&self, path: &str, len: u64) -> Self::Completion>; /// Write `buf` to `fd` at `offset`. /// @@ -120,7 +120,7 @@ pub trait SpacetimeIO { fd: Self::Fd, buf: B, offset: u64, - ) -> impl Future>>; + ) -> Self::Completion>>; /// Read `size_of::()` bytes from `fd` at `offset` and interpret them at /// type `B`. @@ -136,15 +136,15 @@ pub trait SpacetimeIO { fd: Self::Fd, buf: B, offset: u64, - ) -> impl Future>>; + ) -> Self::Completion>>; /// Call `fsync(2)` on `fd`. - fn fsync(&self, fd: Self::Fd) -> impl Future>; + fn fsync(&self, fd: Self::Fd) -> Self::Completion>; /// Call `fdatasync(2)` on `fd`. - fn fdatasync(&self, fd: Self::Fd) -> impl Future>; + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion>; /// Allocate `additional` bytes for the file `fd`. - fn reserve(&self, fd: Self::Fd, additional: u64) -> impl Future>; + fn reserve(&self, fd: Self::Fd, additional: u64) -> Self::Completion>; /// Determine the length of the file `fd`. /// diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index c81cb5ca270..a86d4b0834a 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -3,7 +3,12 @@ use alloc::{ collections::{BTreeMap, VecDeque}, sync::Arc, }; -use core::{num::NonZeroUsize, result::Result}; +use core::{ + num::NonZeroUsize, + pin::Pin, + result::Result, + task::{Context, Poll}, +}; use futures_channel::oneshot; use crate::{ @@ -13,7 +18,6 @@ use crate::{ mod fs; pub mod op; -use op::{Completion, Submission}; pub use crate::io::SECTOR_SIZE; pub use fs::File; @@ -77,127 +81,118 @@ impl SimulatorIO { } /// Execute `sqe`. - pub fn execute(&self, sqe: Box) { + pub fn execute(&self, sqe: Box) { self.inner.lock().execute(sqe); } /// Remove and return the submission at the front of the queue, if any. - pub fn next_submission(&self) -> Option> { + pub fn next_submission(&self) -> Option> { self.inner.lock().next_submission() } /// Remove and return a random submission, or `None` if the queue is empty. - pub fn random_submission(&self, rng: &Rng) -> Option> { + pub fn random_submission(&self, rng: &Rng) -> Option> { self.inner.lock().random_submission(rng) } /// Remove and return the completion at the front of the queue, if any. - pub fn next_completion(&self) -> Option> { + pub fn next_completion(&self) -> Option> { self.inner.lock().next_completion() } /// Remove and return a random completion, or `None` if the queue is empty. - pub fn random_completion(&self, rng: &Rng) -> Option> { + pub fn random_completion(&self, rng: &Rng) -> Option> { self.inner.lock().random_completion(rng) } - async fn submit_and_wait( - &self, - op: impl FnOnce(oneshot::Sender) -> Box, - ) -> Result { + fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> Completion { let (tx, rx) = oneshot::channel(); self.inner.lock().submit(op(tx)); - rx.await + Completion(rx) + } + + fn submit_all>>( + &self, + ops: impl FnOnce(oneshot::Sender) -> I, + ) -> Completion { + let (tx, rx) = oneshot::channel(); + let mut inner = self.inner.lock(); + ops(tx).for_each(|op| inner.submit(op)); + Completion(rx) + } +} + +#[must_use = "completions must be polled to completion"] +pub struct Completion(oneshot::Receiver); + +impl Future for Completion { + type Output = T; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.as_mut().0).poll(cx).map(Result::unwrap) } } impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; + type Completion = Completion; - async fn open_file(&self, path: &str) -> Result { - self.submit_and_wait(|tx| op::open_file(path, tx)) - .await - .expect("`open_file` future cancelled") + fn open_file(&self, path: &str) -> Self::Completion> { + self.submit(op::open_file(path)) } - async fn create_file(&self, path: &str, len: u64) -> Result { - self.submit_and_wait(|tx| op::create_file(path, len, tx)) - .await - .expect("`create_file` future cancelled") + fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + self.submit(op::create_file(path, len)) } - async fn write_all_at( + fn write_all_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { + ) -> Self::Completion>> { let () = B::ASSERT_VALID_LAYOUT; if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit_and_wait(|tx| { - op::ready( - Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }), - tx, - ) - }) - .await - .expect("`write_all_at` future cancelled") + self.submit(op::ready(Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }))) } else { - let (tx, rx) = oneshot::channel(); - for op in op::write_at(fd, buf, offset, tx) { - self.inner.lock().submit(op); - } - rx.await.expect("`write_all_at` future cancelled") + self.submit_all(op::write_at(fd, buf, offset)) } } - async fn read_exact_at( + fn read_exact_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { + ) -> Self::Completion>> { let () = B::ASSERT_VALID_LAYOUT; if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit_and_wait(|tx| { - op::ready( - Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }), - tx, - ) - }) - .await - .expect("`read_exact_at` future cancelled") + self.submit(op::ready(Err(ErrorWith { + error: fs::Error::UnalignedOffset.into(), + with: buf, + }))) } else { - let (tx, rx) = oneshot::channel(); - for op in op::read_at(fd, buf, offset, tx) { - self.inner.lock().submit(op); - } - rx.await.expect("`read_exact_at` future cancelled") + self.submit_all(op::read_at(fd, buf, offset)) } } - async fn fsync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { - Ok(()) + fn fsync(&self, _fd: Self::Fd) -> Self::Completion> { + self.submit(op::ready(Ok(()))) } - async fn fdatasync(&self, _fd: Self::Fd) -> Result<(), Self::Error> { - Ok(()) + fn fdatasync(&self, _fd: Self::Fd) -> Self::Completion> { + self.submit(op::ready(Ok(()))) } - async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let len = self.length(fd.clone()).await?; - self.submit_and_wait(|tx| op::set_len(fd, len + additional, tx)) - .await - .expect("`set_len` future cancelled") + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { + assert!(total >= fd.len()); + self.submit(op::set_len(fd, total)) } fn length(&self, fd: Self::Fd) -> Self::Completion> { @@ -208,8 +203,8 @@ impl SpacetimeIO for SimulatorIO { #[derive(Default)] struct SimulatorIOInner { files: BTreeMap, fs::File>, - submissions: VecDeque>, - completions: VecDeque>, + submissions: VecDeque>, + completions: VecDeque>, } impl SimulatorIOInner { @@ -229,31 +224,31 @@ impl SimulatorIOInner { progress } - fn execute(&mut self, sqe: Box) { + fn execute(&mut self, sqe: Box) { if let Some(cqe) = sqe.execute(&mut self.files) { self.completions.push_back(cqe); } } - fn next_submission(&mut self) -> Option> { + fn next_submission(&mut self) -> Option> { self.submissions.pop_front() } - fn random_submission(&mut self, rng: &Rng) -> Option> { + fn random_submission(&mut self, rng: &Rng) -> Option> { let len = NonZeroUsize::new(self.submissions.len())?; self.submissions.remove(rng.index(len.get())) } - fn next_completion(&mut self) -> Option> { + fn next_completion(&mut self) -> Option> { self.completions.pop_front() } - fn random_completion(&mut self, rng: &Rng) -> Option> { + fn random_completion(&mut self, rng: &Rng) -> Option> { let len = NonZeroUsize::new(self.completions.len())?; self.completions.remove(rng.index(len.get())) } - fn submit(&mut self, op: Box) { + fn submit(&mut self, op: Box) { self.submissions.push_back(op); } } diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs index 3c2b1f90086..0e855313819 100644 --- a/crates/runtime-core/src/sim/io/op.rs +++ b/crates/runtime-core/src/sim/io/op.rs @@ -1,4 +1,4 @@ -use core::any::Any; +use core::{any::Any, iter::Scan, ops::Range}; use alloc::{ boxed::Box, @@ -37,6 +37,7 @@ pub trait Submission: Send + Any { /// An object containing the result of executing a [Submission], as well as a /// handle to resolve a future waiting on the outcome of the operation. pub trait Completion: Send { + fn success(&self) -> bool; /// Resolve the future waiting on the outcome of the operation. fn complete(self: Box); } @@ -45,6 +46,10 @@ pub trait Completion: Send { /// operation. pub type OnComplete = oneshot::Sender; +pub type ScanState = (fs::File, usize, Arc>>); +pub type PageWrites = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; +pub type PageReads = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; + pub type WriteAtResult = Result>; /// Write the contents of `buf` to `fd` at `offset`. @@ -57,28 +62,29 @@ pub fn write_at( fd: fs::File, buf: B, offset: u64, - on_complete: OnComplete>, -) -> impl Iterator> { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).map(move |buf_page| { - let op = WritePage { - fd: fd.clone(), - file_page: first_page + buf_page, - buf_page, - state: state.clone(), - }; +) -> impl FnOnce(OnComplete>) -> PageWrites { + move |on_complete| { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Arc::new(spin::Mutex::new(PagedOpState { + buf: Some(buf), + on_complete: Some(on_complete), + remaining: page_count, + first_error: None, + })); + + (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { + let op = WritePage { + fd: fd.clone(), + file_page: *first_page + buf_page, + buf_page, + state: state.clone(), + }; - Box::new(op) as Box - }) + Some(Box::new(op)) + }) + } } pub type ReadAtResult = Result>; @@ -94,67 +100,81 @@ pub fn read_at( fd: fs::File, buf: B, offset: u64, - on_complete: OnComplete>, -) -> impl Iterator> { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).map(move |buf_page| { - let op = ReadPage { - fd: fd.clone(), - file_page: first_page + buf_page, - buf_page, - state: state.clone(), - }; +) -> impl FnOnce(OnComplete>) -> PageReads { + move |on_complete| { + let first_page = (offset / SECTOR_SIZE as u64) as usize; + let page_count = buf.as_bytes().len() / SECTOR_SIZE; + + let state = Arc::new(spin::Mutex::new(PagedOpState { + buf: Some(buf), + on_complete: Some(on_complete), + remaining: page_count, + first_error: None, + })); + + (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { + let op = ReadPage { + fd: fd.clone(), + file_page: *first_page + buf_page, + buf_page, + state: state.clone(), + }; - Box::new(op) as Box - }) + Some(Box::new(op)) + }) + } } /// Open file at `path`. -pub fn open_file(path: &str, on_complete: OnComplete>) -> Box { - Box::new(OpenFile { - path: path.into(), - on_complete, - }) +pub fn open_file(path: &str) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| { + Box::new(OpenFile { + path: path.into(), + on_complete, + }) + } } /// Create a new file at `path` and allocate `len` space for it. -pub fn create_file(path: &str, len: u64, on_complete: OnComplete>) -> Box { - Box::new(CreateFile { - path: path.into(), - len, - on_complete, - }) +pub fn create_file(path: &str, len: u64) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| { + Box::new(CreateFile { + path: path.into(), + len, + on_complete, + }) + } } /// Get the length of the file `fd`. -pub fn get_len(fd: fs::File, on_complete: OnComplete>) -> Box { - Box::new(GetLen { fd, on_complete }) +pub fn get_len(fd: fs::File) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| Box::new(GetLen { fd, on_complete }) } /// Set the length of the file `fd`. -pub fn set_len(fd: fs::File, len: u64, on_complete: OnComplete>) -> Box { - Box::new(SetLen { fd, len, on_complete }) +pub fn set_len(fd: fs::File, len: u64) -> impl FnOnce(OnComplete>) -> Box { + move |on_complete| Box::new(SetLen { fd, len, on_complete }) } struct GenericCompletion { + success: bool, result: T, on_complete: OnComplete, } -fn completion(result: T, on_complete: OnComplete) -> Box { - Box::new(GenericCompletion { result, on_complete }) +fn completion(success: bool, result: T, on_complete: OnComplete) -> Box { + Box::new(GenericCompletion { + success, + result, + on_complete, + }) } impl Completion for GenericCompletion { + fn success(&self) -> bool { + self.success + } + fn complete(self: Box) { let Self { result, on_complete, .. @@ -200,8 +220,107 @@ impl Submission for Ready { } /// An operation that is already complete with `result`. -pub fn ready(result: T, on_complete: OnComplete) -> Box { - Box::new(Ready(completion(result, on_complete))) +pub fn ready(result: T) -> impl FnOnce(OnComplete) -> Box { + move |on_complete| Box::new(Ready(completion(true, result, on_complete))) +} + +/// [Submission] created by [link]. +pub(crate) struct SoftLink { + a: Box, + b: Box, +} + +impl Submission for SoftLink { + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { a, b } = *self; + let result_a = a.execute(files); + let result_b = if result_a.as_ref().is_none_or(|result| result.success()) { + b.execute(files) + } else { + b.cancel() + }; + + Some(Box::new(LinkedCompletion { + a: result_a, + b: result_b, + })) + } + + fn cancel(self: Box) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.cancel(), + b: b.cancel(), + })) + } +} + +/// Link `a` and `b`, such that `b` gets executed after `a`. +/// +/// If `a` fails (i.e. its [Completion::success] returns `false`), `b` is +/// cancelled. +/// +/// Corresponds to io-uring's `IOSQE_IO_LINK` flag. To emulate +/// `IOSQE_IO_HARDLINK`, see [hard_link]. +pub fn link(a: Box, b: Box) -> Box { + Box::new(SoftLink { a, b }) +} + +/// [Submission] created by [hard_link]. +pub(crate) struct HardLink { + a: Box, + b: Box, +} + +impl Submission for HardLink { + fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.execute(files), + b: b.execute(files), + })) + } + + fn cancel(self: Box) -> Option> { + let Self { a, b } = *self; + Some(Box::new(LinkedCompletion { + a: a.cancel(), + b: b.cancel(), + })) + } +} + +/// Link `a` and `b`, such that `b` gets executed after `a`. +/// +/// Unlike [link], this executes both submissions regardless of the result. It +/// just enforces the ordering constraint that `b` will never execute before +/// `a`. +/// +/// Corresponds to io-uring's `IOSQE_IO_HARDLINK` flag. To emulate +/// `IOSQE_IO_LINK`, see [link]. +pub fn hard_link(a: Box, b: Box) -> Box { + Box::new(HardLink { a, b }) +} + +struct LinkedCompletion { + a: Option>, + b: Option>, +} + +impl Completion for LinkedCompletion { + fn success(&self) -> bool { + self.a.as_ref().is_none_or(|result| result.success()) && self.b.as_ref().is_none_or(|result| result.success()) + } + + fn complete(self: Box) { + let Self { a, b } = *self; + if let Some(a) = a { + a.complete(); + } + if let Some(b) = b { + b.complete(); + } + } } /// [Submission] created by [open_file]. @@ -214,12 +333,12 @@ impl Submission for OpenFile { fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { let Self { path, on_complete } = *self; let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { let Self { path: _, on_complete } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -241,7 +360,7 @@ impl Submission for CreateFile { file.set_len(len)?; Ok(file) })(); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { @@ -250,7 +369,7 @@ impl Submission for CreateFile { len: _, on_complete, } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -264,12 +383,12 @@ impl Submission for GetLen { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, on_complete } = *self; let result = Ok(fd.len()); - Some(completion(result, on_complete)) + Some(completion(true, result, on_complete)) } fn cancel(self: Box) -> Option> { let Self { fd: _, on_complete } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } @@ -284,7 +403,7 @@ impl Submission for SetLen { fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { let Self { fd, len, on_complete } = *self; let result = fd.set_len(len).map_err(Error::from); - Some(completion(result, on_complete)) + Some(completion(result.is_ok(), result, on_complete)) } fn cancel(self: Box) -> Option> { @@ -293,11 +412,11 @@ impl Submission for SetLen { len: _, on_complete, } = *self; - Some(completion(Err(Error::Cancelled), on_complete)) + Some(completion(false, Err(Error::Cancelled), on_complete)) } } -struct PagedOpState { +pub struct PagedOpState { buf: Option, on_complete: Option>>>, remaining: usize, @@ -329,6 +448,10 @@ struct PageOpCompletion { } impl Completion for PageOpCompletion { + fn success(&self) -> bool { + self.state.lock().first_error.is_none() + } + fn complete(self: Box) { let (on_complete, result) = { let mut state = self.state.lock(); diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index d48e6623a62..1aeb31c0af1 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,15 +5,9 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -#[cfg(unix)] -use std::os::unix::fs::FileExt as _; -#[cfg(windows)] -use std::os::windows::fs::FileExt as _; - use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; use static_assertions::assert_not_impl_any; -use tokio::fs::OpenOptions; -use tokio::{runtime, task::spawn_blocking}; +use tokio::runtime; /// Implementation of [SpacetimeIO] that runs on a tokio runtime. pub struct TokioIO { @@ -33,6 +27,38 @@ impl TokioIO { assert_not_impl_any!(TokioIO: Send); +#[must_use = "completions must be polled to completion"] +pub struct Completion(tokio::task::JoinHandle); + +impl Future for Completion { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match Pin::new(&mut this.0).poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(result) => match result { + Ok(output) => Poll::Ready(output), + Err(error) => { + if error.is_panic() { + panic::resume_unwind(error.into_panic()) + } else if error.is_cancelled() { + panic!("I/O task unexpectedly cancelled"); + } else { + unreachable!("unexpected I/O task error") + } + } + }, + } + } +} + +impl From> for Completion { + fn from(handle: tokio::task::JoinHandle) -> Self { + Self(handle) + } +} + impl SpacetimeIO for TokioIO { // NOTE: This operates on a [std::fs::File] handle instead of // [tokio::fs::File] because `pwrite`/`pread`-style APIs are not available @@ -41,75 +67,77 @@ impl SpacetimeIO for TokioIO { // here we can avoid some locking. type Fd = Arc; type Error = io::Error; + type Completion = Completion; - async fn open_file(&self, path: &str) -> Result { - let _rt = self.rt.enter(); - - let mut open_options = tokio::fs::File::options(); - open_options.read(true).write(true); - let file = open_with_direct_io(open_options, path).await?; - - Ok(Arc::new(file.into_std().await)) + fn open_file(&self, path: &str) -> Self::Completion> { + let path = PathBuf::from(path); + self.rt + .spawn_blocking(move || { + let mut open_options = std::fs::File::options(); + open_options.read(true).write(true); + platform::open_with_direct_io(open_options, path).map(Arc::new) + }) + .into() } - async fn create_file(&self, path: &str, len: u64) -> Result { - let _rt = self.rt.enter(); - - let mut open_options = tokio::fs::File::options(); - open_options.read(true).write(true).create_new(true); - let file = open_with_direct_io(open_options, path).await?; - file.set_len(len).await?; - - Ok(Arc::new(file.into_std().await)) + fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + let path = PathBuf::from(path); + self.rt + .spawn_blocking(move || { + let mut open_options = std::fs::File::options(); + open_options.read(true).write(true).create_new(true); + let file = platform::open_with_direct_io(open_options, path)?; + file.set_len(len)?; + Ok(Arc::new(file)) + }) + .into() } - async fn write_all_at( + fn write_all_at( &self, fd: Self::Fd, buf: B, offset: u64, - ) -> Result> { - let _rt = self.rt.enter(); - asyncify(move || match write_all_at(&fd, buf.as_bytes(), offset) { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - }) - .await + ) -> Self::Completion>> { + self.rt + .spawn_blocking(move || match platform::write_all_at(&fd, buf.as_bytes(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + }) + .into() } - async fn read_exact_at( + fn read_exact_at( &self, fd: Self::Fd, mut buf: B, offset: u64, - ) -> Result> { - let _rt = self.rt.enter(); - asyncify(move || match read_exact_at(&fd, buf.as_bytes_mut(), offset) { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), - }) - .await + ) -> Self::Completion>> { + self.rt + .spawn_blocking(move || match platform::read_exact_at(&fd, buf.as_bytes_mut(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + }) + .into() } - async fn fsync(&self, fd: Self::Fd) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || fd.sync_all()).await + fn fsync(&self, fd: Self::Fd) -> Self::Completion> { + self.rt.spawn_blocking(move || fd.sync_all()).into() } - async fn fdatasync(&self, fd: Self::Fd) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || fd.sync_data()).await + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { + self.rt.spawn_blocking(move || fd.sync_data()).into() } - async fn reserve(&self, fd: Self::Fd, additional: u64) -> Result<(), Self::Error> { - let _rt = self.rt.enter(); - asyncify(move || { - let len = fd.metadata()?.len(); - fd.set_len(len + additional)?; - - Ok(()) - }) - .await + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { + self.rt + .spawn_blocking(move || { + let mut fd = fd.try_clone()?; + let len = file_length(&mut fd)?; + assert!(total >= len); + fd.set_len(total) + }) + .into() } fn length(&self, fd: Self::Fd) -> Self::Completion> { @@ -122,87 +150,107 @@ impl SpacetimeIO for TokioIO { } } -async fn asyncify(f: F) -> R -where - F: FnOnce() -> R + Send + 'static, - R: Send + 'static, -{ - spawn_blocking(f).await.unwrap_or_else(|e| match e.try_into_panic() { - Ok(panic_payload) => std::panic::resume_unwind(panic_payload), - // A cancellation should not be possible, because we await the task. - Err(e) => panic!("unexpected error joining blocking task: {e}"), - }) +fn file_length(fd: &mut std::fs::File) -> io::Result { + let pos = fd.stream_position()?; + let len = fd.seek(SeekFrom::End(0))?; + + if pos != len { + fd.seek(SeekFrom::Start(pos))?; + } + + Ok(len) } -#[cfg(all(unix, not(target_os = "macos")))] -async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { - options.custom_flags(libc::O_DIRECT).open(path).await +mod platform { + #[cfg(unix)] + pub use super::unix::*; + + #[cfg(windows)] + pub use super::windows::*; } -#[cfg(target_os = "macos")] -async fn open_with_direct_io(options: OpenOptions, path: &str) -> io::Result { - let file = options.open(path).await?; - asyncify(move || { +#[cfg(unix)] +mod unix { + use std::{ + io, + os::unix::fs::{FileExt as _, OpenOptionsExt as _}, + path::Path, + }; + + #[inline] + pub fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { + fd.read_exact_at(buf, offset) + } + + #[inline] + pub fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { + fd.write_all_at(buf, offset) + } + + #[cfg(not(target_os = "macos"))] + pub fn open_with_direct_io(mut options: std::fs::OpenOptions, path: impl AsRef) -> io::Result { + options.custom_flags(libc::O_DIRECT).open(path) + } + + #[cfg(target_os = "macos")] + pub async fn open_with_direct_io( + options: std::fs::OpenOptions, + path: impl AsRef, + ) -> io::Result { + let file = options.open(path)?; let res = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_NOCACHE, 1) }; if res == -1 { Err(io::Error::last_os_error()) } else { Ok(file) } - }) - .await + } } #[cfg(windows)] -async fn open_with_direct_io(mut options: OpenOptions, path: &str) -> io::Result { - options - .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) - .open(path) - .await -} - -#[cfg(unix)] -#[inline] -fn read_exact_at(fd: &std::fs::File, buf: &mut [u8], offset: u64) -> io::Result<()> { - fd.read_exact_at(buf, offset) -} +mod windows { + use std::io; -#[cfg(windows)] -fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match fd.seek_read(buf, offset) { - Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), - Ok(n) => { - offset += n as u64; - buf = &mut buf[n..]; + pub fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match fd.seek_write(buf, offset) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + offset += n as u64; + buf = &buf[n..]; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), } - } - - Ok(()) -} -#[cfg(unix)] -#[inline] -fn write_all_at(fd: &std::fs::File, buf: &[u8], offset: u64) -> io::Result<()> { - fd.write_all_at(buf, offset) -} + Ok(()) + } -#[cfg(windows)] -fn write_all_at(fd: &std::fs::File, mut buf: &[u8], mut offset: u64) -> io::Result<()> { - while !buf.is_empty() { - match fd.seek_write(buf, offset) { - Ok(0) => return Err(io::ErrorKind::WriteZero.into()), - Ok(n) => { - offset += n as u64; - buf = &buf[n..]; + pub fn read_exact_at(fd: &std::fs::File, mut buf: &mut [u8], mut offset: u64) -> io::Result<()> { + while !buf.is_empty() { + match fd.seek_read(buf, offset) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => { + offset += n as u64; + buf = &mut buf[n..]; + } + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), } - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {} - Err(e) => return Err(e), } + + Ok(()) } - Ok(()) + pub async fn open_with_direct_io( + mut options: std::fs::OpenOptions, + path: impl AsRef, + ) -> io::Result { + use std::os::windows::fs::OpenOptionsExt as _; + + options + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_NO_BUFFERING) + .open(path) + } } From a375c7fa599a234e8fc8c03ec66cbd17d545cdd1 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 27 Aug 2026 10:46:36 +0200 Subject: [PATCH 13/47] WIP: simulator redesign --- Cargo.lock | 2 + crates/runtime-core/Cargo.toml | 7 +- crates/runtime-core/src/io/buf.rs | 178 +++++ crates/runtime-core/src/io/error.rs | 43 + crates/runtime-core/src/io/mod.rs | 95 +-- crates/runtime-core/src/lib.rs | 2 +- crates/runtime-core/src/sim/executor/io.rs | 101 --- crates/runtime-core/src/sim/executor/mod.rs | 37 +- crates/runtime-core/src/sim/io/executor.rs | 844 ++++++++++++++++++++ crates/runtime-core/src/sim/io/fs.rs | 8 +- crates/runtime-core/src/sim/io/mod.rs | 514 ++++++++---- crates/runtime-core/src/sim/io/op.rs | 563 ------------- crates/runtime/src/io/tokio.rs | 12 +- 13 files changed, 1470 insertions(+), 936 deletions(-) create mode 100644 crates/runtime-core/src/io/buf.rs create mode 100644 crates/runtime-core/src/io/error.rs delete mode 100644 crates/runtime-core/src/sim/executor/io.rs create mode 100644 crates/runtime-core/src/sim/io/executor.rs delete mode 100644 crates/runtime-core/src/sim/io/op.rs diff --git a/Cargo.lock b/Cargo.lock index c38b884a6ac..e910f3b4c42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8344,8 +8344,10 @@ version = "2.8.3" dependencies = [ "async-task", "futures-channel", + "slab", "spin", "thiserror 2.0.17", + "tokio", "zerocopy", ] diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 6b644b6ab92..5b2b8571106 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -11,11 +11,16 @@ workspace = true [features] default = [] -sim = ["dep:async-task", "dep:futures-channel", "dep:spin"] +alloc = [] +sim = ["alloc", "dep:async-task", "dep:futures-channel", "dep:slab", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } +slab = { version = "0.4", default-features = false, optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } thiserror = { version = "2.0", default-features = false } zerocopy = "0.8" + +[dev-dependencies] +tokio.workspace = true diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-core/src/io/buf.rs new file mode 100644 index 00000000000..16a8123b5c5 --- /dev/null +++ b/crates/runtime-core/src/io/buf.rs @@ -0,0 +1,178 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::io::SECTOR_SIZE; + +/// Types that can be safely converted to and from sector-aligned byte slices. +pub trait AlignedBytes: Sized { + /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the + /// right alignment. + /// + /// The type must also not rely on drop glue, i.e. `!core::mem::needs_drop()`. + /// + /// NOTE: Associated constants are evaluated lazily -- add a free + /// + /// `const _: () = ::ASSERT_VALID_LAYOUT;` + /// + /// for each `T` that is supposed to be used as an `AlignedBytes`. + const ASSERT_VALID_LAYOUT: () = { + assert!(align_of::() == SECTOR_SIZE); + assert!(size_of::().is_multiple_of(SECTOR_SIZE)); + assert!(!core::mem::needs_drop::()); + }; + + /// Reinterpret `self` as a byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes(&self) -> &[u8]; + + /// Reinterpret `self` as a mutable byte slice. + /// + /// The returned slice will be of length `size_of::()`. + fn as_bytes_mut(&mut self) -> &mut [u8]; + + /// Reinterpret a byte slice as `Self`. + /// + /// The slice must be of length `size_of::()`. + /// + /// NOTE: Any slice of the right size, but consisting of only `0` (zero) + /// bytes can be converted to `Self`. It is the caller's responsibility to + /// validate the returned type as per the application's invariants. + /// + /// # Panics + /// + /// Panics if `b.len() != size_of::()`. + fn from_bytes(b: &[u8]) -> Self; +} + +impl AlignedBytes for T { + fn as_bytes(&self) -> &[u8] { + ::as_bytes(self) + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + ::as_mut_bytes(self) + } + + fn from_bytes(b: &[u8]) -> Self { + Self::read_from_bytes(b).unwrap() + } +} + +#[cfg(feature = "alloc")] +mod boxed { + use alloc::boxed::Box; + use core::{alloc::Layout, any::TypeId, ptr::NonNull}; + + use crate::io::AlignedBytes; + + /// A type-erased [AlignedBytes] heap allocation. + pub struct ErasedBox { + ptr: NonNull, + len: usize, + layout: Layout, + ty: TypeId, + } + + impl ErasedBox { + /// Create an [ErasedBox] from `B` by allocating a new [Box]. + pub fn from_aligned(b: B) -> Self { + Self::from_aligned_box(Box::new(b)) + } + + /// Create an [ErasedBox] from an already-boxed `B`. + pub fn from_aligned_box(b: Box) -> Self { + let () = B::ASSERT_VALID_LAYOUT; + + let ptr = Box::into_raw(b); + Self { + ptr: NonNull::new(ptr.cast()).unwrap(), + len: size_of::(), + layout: Layout::from_size_align(size_of::(), align_of::()).unwrap(), + ty: TypeId::of::(), + } + } + + /// Reify `B` via casting. + pub fn into_aligned(self) -> B { + *Self::into_aligned_box(self) + } + + /// Reify `B` via casting, without unboxing. + pub fn into_aligned_box(self) -> Box { + assert_eq!(self.len, size_of::()); + assert_eq!(self.ty, TypeId::of::()); + + let boxed = unsafe { Box::from_raw(self.ptr.as_ptr().cast::()) }; + // Prevent drop, which would deallocate. + core::mem::forget(self); + + boxed + } + + pub fn as_ptr(&self) -> ErasedBoxPtr { + ErasedBoxPtr { + ptr: self.ptr.as_ptr(), + len: self.len, + } + } + } + + impl Drop for ErasedBox { + fn drop(&mut self) { + unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } + } + } + + pub struct ErasedBoxPtr { + ptr: *mut u8, + len: usize, + } + + impl ErasedBoxPtr { + pub fn as_bytes(&mut self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr, self.len) } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) } + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[repr(C, align(4096))] + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct Trivial([u8; 4096]); + + impl AlignedBytes for Trivial { + fn as_bytes(&self) -> &[u8] { + &self.0 + } + + fn as_bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 + } + + fn from_bytes(b: &[u8]) -> Self { + assert_eq!(b.len(), size_of::()); + let mut a = [0; 4096]; + a.copy_from_slice(b); + Self(a) + } + } + + #[test] + fn roundtrip_preserves_value() { + let t = Trivial([32; 4096]); + + let erased = ErasedBox::from_aligned(t); + let reified = erased.into_aligned::(); + + assert_eq!(reified, t); + } + } +} +#[cfg(feature = "alloc")] +pub use boxed::{ErasedBox, ErasedBoxPtr}; diff --git a/crates/runtime-core/src/io/error.rs b/crates/runtime-core/src/io/error.rs new file mode 100644 index 00000000000..0a06c1ada98 --- /dev/null +++ b/crates/runtime-core/src/io/error.rs @@ -0,0 +1,43 @@ +/// An error `E`, along with auxiliary data `T`. +/// +/// `T` is usually a buffer of type [AlignedBytes], whose ownership is +/// transferred back to the caller when an error occurs. +/// +/// As this type signifies an error condition, the contents of `T` are +/// unspecified. +/// +/// [AlignedBytes]: crate::io::buf::AlignedBytes +#[derive(Debug)] +pub struct ErrorWith { + pub error: E, + pub with: T, +} + +impl ErrorWith { + /// Map a type-changing function over `self.error`. + pub fn map_err(self, f: impl FnOnce(E) -> F) -> ErrorWith { + ErrorWith { + error: f(self.error), + with: self.with, + } + } + + /// Map a type-changing function over `self.with`. + pub fn map_with(self, f: impl FnOnce(T) -> U) -> ErrorWith { + ErrorWith { + error: self.error, + with: f(self.with), + } + } + + /// Extract `self.error`, discarding `self.with`. + pub fn into_err(self) -> E { + self.error + } + + /// Convert from `&ErrorWith` to `ErrorWith<&E, &T>`. + pub fn as_ref(&self) -> ErrorWith<&E, &T> { + let Self { ref error, ref with } = *self; + ErrorWith { error, with } + } +} diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index 7b54b1dd39b..d5a946b14ca 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -1,74 +1,27 @@ -use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +mod buf; +pub use buf::AlignedBytes; +#[cfg(feature = "alloc")] +pub use buf::{ErasedBox, ErasedBoxPtr}; + +mod error; +pub use error::ErrorWith; /// Size in bytes of a disk sector. pub const SECTOR_SIZE: usize = 4096; -/// Types that can be safely converted to and from sector-aligned byte slices. -pub trait AlignedBytes: Sized { - /// Assert that the type' size is a multiple of [SECTOR_SIZE] and has the - /// right aligment. - /// - /// NOTE: Associated constants are evaluated lazily -- add a free - /// - /// `const _: () = ::ASSERT_VALID_LAYOUT;` - /// - /// for each `T` that is supposed to be used as an `AlignedBytes`. - const ASSERT_VALID_LAYOUT: () = { - assert!(align_of::() == SECTOR_SIZE); - assert!(size_of::().is_multiple_of(SECTOR_SIZE)); - }; - - /// Reinterpret `self` as a byte slice. - /// - /// The returned slice will be of length `size_of::()`. - fn as_bytes(&self) -> &[u8]; - - /// Reinterpret `self` as a mutable byte slice. - /// - /// The returned slice will be of length `size_of::()`. - fn as_bytes_mut(&mut self) -> &mut [u8]; - - /// Reinterpret a byte slice as `Self`. - /// - /// The slice must be of length `size_of::()`. - /// - /// NOTE: Any slice of the right size, but consisting of only `0` (zero) - /// bytes can be converted to `Self`. It is the caller's responsibility to - /// validate the returned type as per the application's invariants. - /// - /// # Panics - /// - /// Panics if `b.len() != size_of::()`. - fn from_bytes(b: &[u8]) -> Self; +/// Subset of the `statx` metadata. +#[derive(Debug)] +#[non_exhaustive] +pub struct Statx { + pub size: u64, } -impl AlignedBytes for T { - fn as_bytes(&self) -> &[u8] { - ::as_bytes(self) - } - - fn as_bytes_mut(&mut self) -> &mut [u8] { - ::as_mut_bytes(self) - } - - fn from_bytes(b: &[u8]) -> Self { - Self::read_from_bytes(b).unwrap() +impl Statx { + pub fn from_size(size: u64) -> Self { + Self { size } } } -/// An error `E`, along with auxiliary data `T`. -/// -/// `T` is usually a buffer of type [AlignedBytes], whose ownership is -/// transferred back to the caller when an error occurs. -/// -/// As this type signifies an error condition, the contents of `T` are -/// unspecified. -#[derive(Debug)] -pub struct ErrorWith { - pub error: E, - pub with: T, -} - /// The canonical, low-level I/O API. /// /// Currently only supports file I/O, but eventually all I/O performed by @@ -98,15 +51,17 @@ pub trait SpacetimeIO { /// from being `no_std`. /// /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 - type Error; + type Error: core::error::Error; + /// The completion [Future] of all methods in this trait. + type Completion: Future + Unpin; /// Open the file at `path`. fn open_file(&self, path: &str) -> Self::Completion>; - /// Create the file at `path` and allocate `len` bytes. + /// Create the file at `path`. /// /// Returns an error if the file already exists. - fn create_file(&self, path: &str, len: u64) -> Self::Completion>; + fn create_file(&self, path: &str) -> Self::Completion>; /// Write `buf` to `fd` at `offset`. /// @@ -143,11 +98,15 @@ pub trait SpacetimeIO { /// Call `fdatasync(2)` on `fd`. fn fdatasync(&self, fd: Self::Fd) -> Self::Completion>; - /// Allocate `additional` bytes for the file `fd`. - fn reserve(&self, fd: Self::Fd, additional: u64) -> Self::Completion>; + /// Allocate `total` bytes for the file `fd`. + /// + /// Implementations must ensure that attempts to shrink the file result in + /// an error. The operation should succeed if the file's size is already + /// `total`. + fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion>; /// Determine the length of the file `fd`. /// /// This should not depend on `fsync`, i.e. `statx`. See `std::io::Seek::stream_len`. - fn length(&self, fd: Self::Fd) -> Self::Completion>; + fn statx(&self, fd: Self::Fd) -> Self::Completion>; } diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index e35d042ea9a..8a841c22036 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -#[cfg(feature = "sim")] +#[cfg(any(feature = "sim", feature = "alloc"))] extern crate alloc; #[cfg(test)] extern crate std; diff --git a/crates/runtime-core/src/sim/executor/io.rs b/crates/runtime-core/src/sim/executor/io.rs deleted file mode 100644 index ca1831978bf..00000000000 --- a/crates/runtime-core/src/sim/executor/io.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::sim::{io::SimulatorIO, Rng}; - -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Config { - /// The max number of submissions to run per [Driver::tick]. - pub max_submissions_per_tick: usize, - /// The max number of completions to finish per [Driver::tick]. - pub max_completions_per_tick: usize, - /// Submission reordering probability. - /// - /// Describes the probability by which to select the next submission queue - /// entry randomly, as opposed to the oldest entry in the queue. - pub prob_reorder_submissions: f64, - /// Completion reordering probability. - /// - /// Describes the probability by which to select the next completion queue - /// entry randomly, as opposed to the oldest entry in the queue. - pub prob_reorder_completions: f64, - /// Probability by which to skip one submission queue entry. - /// - /// If skipped, the entry still counts towards `max_submissions_per_tick`. - pub prob_skip: f64, - /// Probability by which to cancel a submission queue entry. - /// - /// [crate::sim::io::op::Submission::cancel()] is called on the entry, which - /// may generate a completion. - pub prob_cancel: f64, -} - -impl Default for Config { - fn default() -> Self { - Self { - max_submissions_per_tick: 1, - max_completions_per_tick: 1, - prob_reorder_submissions: 0.0, - prob_reorder_completions: 0.0, - prob_skip: 0.0, - prob_cancel: 0.0, - } - } -} - -pub struct Driver { - io: SimulatorIO, - config: Config, -} - -impl Driver { - pub fn new(config: Config) -> Self { - Self { - io: <_>::default(), - config, - } - } - - /// Advance the I/O simulator according the [Config]. - /// - /// Returns `true` if progress has been made, or there are pending entries - /// in either the submission or completion queue. - pub fn tick(&self, rng: &Rng) -> bool { - let mut progress = false; - for _ in 0..self.config.max_submissions_per_tick { - if !rng.buggify_with_prob(self.config.prob_skip) { - let sqe = if rng.buggify_with_prob(self.config.prob_reorder_submissions) { - self.io.random_submission(rng) - } else { - self.io.next_submission() - }; - - if let Some(sqe) = sqe { - if rng.buggify_with_prob(self.config.prob_cancel) { - sqe.cancel(); - } else { - self.io.execute(sqe); - } - progress = true; - } - } - } - - for _ in 0..self.config.max_completions_per_tick { - let cqe = if rng.buggify_with_prob(self.config.prob_reorder_completions) { - self.io.random_completion(rng) - } else { - self.io.next_completion() - }; - - if let Some(cqe) = cqe { - cqe.complete(); - progress = true - } - } - - progress |= self.io.pending(); - progress - } - - pub fn io(&self) -> &SimulatorIO { - &self.io - } -} diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index 1913329b2e2..a0fbca1bf7c 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -10,12 +10,8 @@ use core::{ use spin::Mutex; -use crate::sim::io::SimulatorIO; - use super::{time::TimeHandle, Rng}; -mod io; - mod task; use task::Abortable; pub use task::{AbortHandle, JoinError, JoinHandle}; @@ -25,23 +21,11 @@ type Runnable = async_task::Runnable; #[derive(Clone, Copy, Debug, PartialEq)] pub struct RuntimeConfig { pub seed: u64, - pub io: Option, } impl RuntimeConfig { pub const fn new(seed: u64) -> Self { - Self { seed, io: None } - } - - pub fn enable_io(self) -> Self { - Self { - io: Some(self.io.unwrap_or_default()), - ..self - } - } - - pub fn with_io_config(self, io: Option) -> Self { - Self { io, ..self } + Self { seed } } } @@ -161,12 +145,6 @@ impl Runtime { } } - // TODO: This is a stopgap to allow submission of I/O tasks. We probably - // want the user-facing API to hide this. - pub fn io(&self) -> Option<&SimulatorIO> { - self.executor.io.as_ref().map(|driver| driver.io()) - } - /// Drive a top-level future to completion on the simulation executor. /// /// While the future runs, spawned tasks share the same deterministic @@ -382,7 +360,6 @@ struct Executor { next_node: AtomicU64, rng: Rng, time: TimeHandle, - io: Option, } impl Executor { @@ -398,7 +375,6 @@ impl Executor { next_node: AtomicU64::new(1), rng: Rng::new(config.seed), time: TimeHandle::new(), - io: config.io.map(io::Driver::new), } } @@ -515,7 +491,6 @@ impl Executor { loop { self.run_all_ready(); - let pending_io = self.drive_io(); if task.is_finished() { let waker = Waker::noop(); return match Pin::new(&mut task).poll(&mut Context::from_waker(waker)) { @@ -524,7 +499,7 @@ impl Executor { }; } - if self.time.wake_next_timer() || pending_io { + if self.time.wake_next_timer() { continue; } @@ -552,14 +527,6 @@ impl Executor { } } - fn drive_io(&self) -> bool { - if let Some(io) = &self.io { - io.tick(&self.rng) - } else { - false - } - } - /// Look up the record for a node, panicking if the node is unknown. fn node_record(&self, node: NodeId) -> Arc { self.nodes diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs new file mode 100644 index 00000000000..9135fa45c7c --- /dev/null +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -0,0 +1,844 @@ +#![allow(unused)] + +use alloc::{ + boxed::Box, + collections::{btree_map, BTreeMap, VecDeque}, + vec::Vec, +}; +use core::result::Result; + +use crate::{ + io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, + sim::{ + io::{fs, Error, Instant}, + Rng, + }, +}; + +#[derive(Clone, Copy)] +pub enum LinkKind { + Soft, + Hard, +} + +pub struct Sqe { + inner: SqeInner, + link: Option, + user_data: Option, +} + +impl Sqe { + pub fn link(mut self, kind: Option) -> Self { + self.link = kind; + self + } + + pub fn is_linked(&self) -> bool { + self.link.is_some() + } + + pub fn attach(mut self, user_data: T) -> Self { + self.user_data.replace(user_data); + self + } + + pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Write { fd, buf, offset }.into() + } + + pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Read { fd, buf, offset }.into() + } + + pub fn open(path: impl AsRef) -> Self { + Open { + path: path.as_ref().into(), + } + .into() + } + + pub fn create(path: impl AsRef) -> Self { + Create { + path: path.as_ref().into(), + } + .into() + } + + pub fn stat(fd: fs::File) -> Self { + Stat { fd }.into() + } + + pub fn fallocate(fd: fs::File, len: u64) -> Self { + Fallocate { fd, total_len: len }.into() + } + + pub fn fsync(fd: fs::File) -> Self { + Fsync { fd }.into() + } + + pub fn fdatasync(fd: fs::File) -> Self { + Fdatasync { fd }.into() + } + + pub fn noop() -> Self { + SqeInner::Noop.into() + } +} + +impl> From for Sqe { + fn from(inner: U) -> Self { + Self { + inner: inner.into(), + link: None, + user_data: None, + } + } +} + +enum SqeInner { + Write(Write), + Read(Read), + Open(Open), + Create(Create), + Stat(Stat), + Fallocate(Fallocate), + Fsync(Fsync), + Fdatasync(Fdatasync), + Noop, +} + +impl SqeInner { + fn cancel(self, user_data: Option) -> Cqe { + match self { + SqeInner::Write(..) => Cqe::Write { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Read(..) => Cqe::Read { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Open(..) => Cqe::Open { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Create(..) => Cqe::Create { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Stat(..) => Cqe::Stat { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fallocate(..) => Cqe::Fallocate { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fsync(..) => Cqe::Fsync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fdatasync(..) => Cqe::Fdatasync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Noop => Cqe::Noop { + result: Err(Error::Cancelled), + user_data, + }, + } + } + + fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { + match self { + SqeInner::Write(mut sqe) => { + let Write { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Operation::WriteSector { + sqe: sqe_id, + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }) + .collect::>(); + let op_count = ops.len(); + let write = InFlightInner::Write { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (write, ops) + } + SqeInner::Read(mut sqe) => { + let Read { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Operation::ReadSector { + sqe: sqe_id, + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }) + .collect::>(); + let op_count = ops.len(); + let read = InFlightInner::Read { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (read, ops) + } + SqeInner::Open(sqe) => ( + InFlightInner::Open { sqe }, + alloc::vec![Operation::Open { sqe: sqe_id }], + ), + SqeInner::Create(sqe) => ( + InFlightInner::Create { sqe }, + alloc::vec![Operation::Create { sqe: sqe_id }], + ), + SqeInner::Stat(sqe) => ( + InFlightInner::Stat { sqe }, + alloc::vec![Operation::Stat { sqe: sqe_id }], + ), + SqeInner::Fallocate(sqe) => ( + InFlightInner::Fallocate { sqe }, + alloc::vec![Operation::Fallocate { sqe: sqe_id }], + ), + SqeInner::Fsync(sqe) => ( + InFlightInner::Fsync { sqe }, + alloc::vec![Operation::Fsync { sqe: sqe_id }], + ), + SqeInner::Fdatasync(sqe) => ( + InFlightInner::Fdatasync { sqe }, + alloc::vec![Operation::Fdatasync { sqe: sqe_id }], + ), + SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), + } + } +} + +impl From for SqeInner { + fn from(inner: Write) -> Self { + Self::Write(inner) + } +} + +impl From for SqeInner { + fn from(inner: Read) -> Self { + Self::Read(inner) + } +} + +impl From for SqeInner { + fn from(inner: Open) -> Self { + Self::Open(inner) + } +} + +impl From for SqeInner { + fn from(inner: Create) -> Self { + Self::Create(inner) + } +} + +impl From for SqeInner { + fn from(inner: Stat) -> Self { + Self::Stat(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fallocate) -> Self { + Self::Fallocate(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fsync) -> Self { + Self::Fsync(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fdatasync) -> Self { + Self::Fdatasync(inner) + } +} + +struct Write { + fd: fs::File, + buf: ErasedBoxPtr, + offset: u64, +} + +struct Read { + fd: fs::File, + buf: ErasedBoxPtr, + offset: u64, +} + +struct Open { + path: Box, +} + +struct Create { + path: Box, +} + +struct Stat { + fd: fs::File, +} + +struct Fallocate { + fd: fs::File, + total_len: u64, +} + +struct Fsync { + #[allow(unused)] + fd: fs::File, +} + +struct Fdatasync { + #[allow(unused)] + fd: fs::File, +} + +#[derive(Debug)] +pub enum Cqe { + Write { + result: Result, + user_data: Option, + }, + Read { + result: Result, + user_data: Option, + }, + Open { + result: Result, + user_data: Option, + }, + Create { + result: Result, + user_data: Option, + }, + Stat { + result: Result, + user_data: Option, + }, + Fallocate { + result: Result<(), Error>, + user_data: Option, + }, + Fsync { + result: Result<(), Error>, + user_data: Option, + }, + Fdatasync { + result: Result<(), Error>, + user_data: Option, + }, + Noop { + result: Result<(), Error>, + user_data: Option, + }, +} + +impl Cqe { + pub fn user_data(&self) -> &Option { + match self { + Self::Write { user_data, .. } + | Self::Read { user_data, .. } + | Self::Open { user_data, .. } + | Self::Create { user_data, .. } + | Self::Stat { user_data, .. } + | Self::Fallocate { user_data, .. } + | Self::Fsync { user_data, .. } + | Self::Fdatasync { user_data, .. } + | Self::Noop { user_data, .. } => user_data, + } + } +} + +type SqeId = usize; + +enum Operation { + WriteSector { + sqe: SqeId, + page_offset: usize, + buf_offset: usize, + }, + ReadSector { + sqe: SqeId, + page_offset: usize, + buf_offset: usize, + }, + Open { + sqe: SqeId, + }, + Create { + sqe: SqeId, + }, + Stat { + sqe: SqeId, + }, + Fallocate { + sqe: SqeId, + }, + Fsync { + sqe: SqeId, + }, + Fdatasync { + sqe: SqeId, + }, + Noop { + sqe: SqeId, + }, +} + +struct Blocked { + link: LinkKind, + sqe: SqeInner, + user_data: Option, +} + +struct InFlight { + inner: InFlightInner, + blocked: VecDeque>, + user_data: Option, +} + +enum InFlightInner { + Write { + sqe: Write, + op_count: usize, + results: Vec>, + }, + Read { + sqe: Read, + op_count: usize, + results: Vec>, + }, + Open { + sqe: Open, + }, + Create { + sqe: Create, + }, + Stat { + sqe: Stat, + }, + Fallocate { + sqe: Fallocate, + }, + Fsync { + sqe: Fsync, + }, + Fdatasync { + sqe: Fdatasync, + }, + Noop, +} + +pub enum WriteFault { + /// Misdirect the write to an arbitrary page offset in the file. + Misdirected { page_offset: usize }, + /// Report the write as successful, but don't write anything. + Lost, + /// Report the write as successful, but write less bytes than requested. + Short { write_bytes: usize }, + /// Delay the write until at least `deadline`. + Delayed { deadline: Instant }, + /// Execute the side effects, but never report completion. + NoCompletion, + /// Report an error without executing side effects. + Error(Error), +} + +pub trait FaultInjector { + fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; +} + +pub struct Executor { + submissions: VecDeque>, + completions: VecDeque>, + + in_flight: [Option>; MAX_INFLIGHT], + executing: VecDeque, + + fstree: BTreeMap, fs::File>, +} + +impl Executor { + pub fn with_capacity(capacity: usize) -> Self { + Self { + submissions: VecDeque::with_capacity(capacity), + completions: VecDeque::with_capacity(capacity), + in_flight: core::array::from_fn(|_| None), + executing: VecDeque::new(), + fstree: BTreeMap::new(), + } + } + + pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> + where + Batch: IntoIterator>, + Batch::IntoIter: ExactSizeIterator, + { + let sqes = sqes.into_iter(); + if self.submissions.len() + sqes.len() >= self.submissions.capacity() { + Err(sqes) + } else { + self.submissions.extend(sqes); + Ok(()) + } + } + + fn complete(&mut self, cqe: Cqe) { + assert!( + self.completions.len() < self.completions.capacity(), + "completion queue overflow" + ); + self.completions.push_back(cqe); + } + + pub fn completed(&mut self) -> impl Iterator> { + self.completions.drain(..) + } + + pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { + let mut progress = self.schedule(); + progress |= self.execute(rng, now); + progress + } + + fn schedule(&mut self) -> bool { + let mut progress = false; + + // Fill free execution slots. + for (id, slot) in self.in_flight.iter_mut().filter(|f| f.is_none()).enumerate() { + let Some(sqe) = self.submissions.pop_front() else { + break; + }; + + // If the sqe is linked, pop the whole chain. + // Links of sqes not submitted in the same batch are ignored. + let mut successors = VecDeque::new(); + if let Some(link) = sqe.link { + let mut link_kind = link; + while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { + successors.push_back(Blocked { + link: link_kind, + sqe: inner, + user_data, + }); + match link { + Some(kind) => link_kind = kind, + None => break, + } + } + } + + let (in_flight, ops) = sqe.inner.schedule(id); + self.executing.extend(ops); + slot.replace(InFlight { + inner: in_flight, + blocked: successors, + user_data: sqe.user_data, + }); + + progress = true + } + + progress + } + + fn execute(&mut self, rng: &Rng, now: Instant) -> bool { + if self.executing.is_empty() { + return false; + } + if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { + match op { + Operation::WriteSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { + let InFlight { + inner: + InFlightInner::Write { + sqe: Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &buf.as_bytes()[buf_offset..end]; + let result = fd.write_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Write { + sqe: Write { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Write { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + Operation::ReadSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { + let InFlight { + inner: + InFlightInner::Read { + sqe: Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + let result = fd.read_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Read { + sqe: Read { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invlid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + Operation::Open { sqe } => { + let InFlight { + inner: InFlightInner::Open { sqe: Open { path } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected open") + }; + + let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Create { sqe } => { + let InFlight { + inner: InFlightInner::Create { sqe: Create { path } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected create") + }; + + let result = match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Stat { sqe } => { + let InFlight { + inner: InFlightInner::Stat { sqe: Stat { fd } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected stat") + }; + + self.complete(Cqe::Stat { + result: Ok(Statx { size: fd.len() }), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fallocate { sqe } => { + let InFlight { + inner: + InFlightInner::Fallocate { + sqe: Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fallocate") + }; + + self.complete(Cqe::Fallocate { + result: fd.set_len(total_len).map_err(Error::from), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fsync { sqe } => { + // TODO: Do something fallible with fd. + let InFlight { + inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fsync") + }; + + self.complete(Cqe::Fsync { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fdatasync { sqe } => { + // TODO: Do something fallible with fd. + let InFlight { + inner: + InFlightInner::Fdatasync { + sqe: Fdatasync { fd: _ }, + }, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fdatasync") + }; + + self.complete(Cqe::Fdatasync { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Noop { sqe } => { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight[sqe].take().expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected noop") + }; + + self.complete(Cqe::Noop { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + } + + return true; + } + + false + } + + fn schedule_linked( + &mut self, + in_flight_slot: SqeId, + prev_succeeded: bool, + mut blocked: VecDeque>, + ) { + if let Some(Blocked { + link, + sqe: next, + user_data, + }) = blocked.pop_front() + { + match (link, prev_succeeded) { + (LinkKind::Soft, false) => { + self.complete(next.cancel(user_data)); + for Blocked { + link: _, + sqe: next, + user_data, + } in blocked + { + self.complete(next.cancel(user_data)); + } + } + (LinkKind::Soft, true) | (LinkKind::Hard, _) => { + let (inner, ops) = next.schedule(in_flight_slot); + self.executing.extend(ops); + self.in_flight[in_flight_slot].replace(InFlight { + inner, + blocked, + user_data, + }); + } + } + } + } +} diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index efa3d9ab755..93e1288fac3 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,6 +1,6 @@ use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cmp, + cmp, fmt, sync::atomic::{AtomicU64, Ordering}, }; @@ -54,6 +54,12 @@ pub struct File { len: Arc, } +impl fmt::Debug for File { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("File").field("len", &self.len).finish() + } +} + impl File { pub(super) fn new() -> Self { Self { diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index a86d4b0834a..d981ba38193 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -1,26 +1,33 @@ -use alloc::{ - boxed::Box, - collections::{BTreeMap, VecDeque}, - sync::Arc, -}; +use alloc::{boxed::Box, sync::Arc}; use core::{ - num::NonZeroUsize, pin::Pin, result::Result, task::{Context, Poll}, + time::Duration, }; use futures_channel::oneshot; +use slab::Slab; use crate::{ - io::{AlignedBytes, ErrorWith, SpacetimeIO}, + io::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}, sim::Rng, }; +mod executor; +use executor::{Cqe, Executor, Sqe}; + mod fs; -pub mod op; +pub use fs::File; pub use crate::io::SECTOR_SIZE; -pub use fs::File; + +/// Simulated clock measurement. +/// +/// In simulated time, an instant is actually a [Duration] since the time +/// instance was instantiated. To avoid confusion, we use the name "instant" to +/// convey that its semantics are that of the standard library type of the same +/// name. +pub type Instant = Duration; #[derive(Debug, thiserror::Error)] pub enum Error { @@ -37,6 +44,8 @@ pub enum Error { /// Injected by the I/O driver. #[error("operation cancelled")] Cancelled, + #[error("submission queue overflow")] + SubmissionQueueOverflow, } impl From for Error { @@ -47,103 +56,239 @@ impl From for Error { #[derive(Clone, Default)] pub struct SimulatorIO { - // TODO: We make `SimulatorIO` `Send + Sync` for now, because - // [crate::sim::executor::Handle] is just `Arc`. This means that a - // future carrying a handle can't be `spawn`ed, because spawning requires - // the future to be `Send`. - // - // We should fix this at some point, so below can become `Rc>`. - inner: Arc>, + inner: Arc, } impl SimulatorIO { - /// Returns `true` if there are entries in either the submission or - /// completion queues. - pub fn pending(&self) -> bool { - let inner = self.inner.lock(); - inner.submissions.len() + inner.completions.len() > 0 - } - - /// Number of entries in the submission queue. - pub fn pending_submissions(&self) -> usize { - self.inner.lock().submissions.len() - } + pub fn tick(&self, rng: &Rng, now: Instant) -> bool { + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let mut buffers = self.inner.buffers.lock(); + + let mut progress = executor.tick(rng, now); + for cqe in executor.completed() { + let completion = pending.remove(cqe.user_data().unwrap()); + match cqe { + Cqe::Write { result, .. } => { + let CompletionHandle::Write { tx, buf_key } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let erased_buf = buffers.remove(buf_key); + let result = match result { + Ok(_written) => Ok(erased_buf), + Err(error) => Err(ErrorWith { + error, + with: erased_buf, + }), + }; + let _ = tx.send(result); + } + Cqe::Read { result, .. } => { + let CompletionHandle::Read { tx, buf_key } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let erased_buf = buffers.remove(buf_key); + let result = match result { + Ok(_written) => Ok(erased_buf), + Err(error) => Err(ErrorWith { + error, + with: erased_buf, + }), + }; + let _ = tx.send(result); + } + Cqe::Open { result, .. } => { + let CompletionHandle::Open { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Create { result, .. } => { + let CompletionHandle::Create { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Stat { result, .. } => { + let CompletionHandle::Stat { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fallocate { result, .. } => { + let CompletionHandle::Fallocate { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fsync { result, .. } => { + let CompletionHandle::Fsync { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Fdatasync { result, .. } => { + let CompletionHandle::Fdatasync { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + Cqe::Noop { result, .. } => { + let CompletionHandle::Noop { tx } = completion else { + unreachable!("invalid cqe / completion pairing") + }; + let _ = tx.send(result); + } + } - /// Number of entries in the completion queue. - pub fn pending_completions(&self) -> usize { - self.inner.lock().completions.len() - } + progress |= true; + } - /// Run the submission at the front of the queue (if any), and complete the - /// completion at the front of the queue (if any). - pub fn tick(&self) -> bool { - self.inner.lock().tick() + progress } +} - /// Execute `sqe`. - pub fn execute(&self, sqe: Box) { - self.inner.lock().execute(sqe); - } +struct SimulatorInner { + executor: spin::Mutex>, + pending: spin::Mutex>, + buffers: Arc>>, +} - /// Remove and return the submission at the front of the queue, if any. - pub fn next_submission(&self) -> Option> { - self.inner.lock().next_submission() +impl Default for SimulatorInner { + fn default() -> Self { + Self { + executor: spin::Mutex::new(Executor::with_capacity(128)), + pending: <_>::default(), + buffers: <_>::default(), + } } +} - /// Remove and return a random submission, or `None` if the queue is empty. - pub fn random_submission(&self, rng: &Rng) -> Option> { - self.inner.lock().random_submission(rng) - } +#[must_use = "completions must be polled to completion"] +pub struct Completion(CompletionInner); - /// Remove and return the completion at the front of the queue, if any. - pub fn next_completion(&self) -> Option> { - self.inner.lock().next_completion() +impl Completion { + pub fn mapped( + rx: oneshot::Receiver>>, + map: fn(Result>) -> T, + ) -> Self { + Self(CompletionInner::Mapped { rx, map }) } +} - /// Remove and return a random completion, or `None` if the queue is empty. - pub fn random_completion(&self, rng: &Rng) -> Option> { - self.inner.lock().random_completion(rng) +impl From> for Completion { + fn from(rx: oneshot::Receiver) -> Self { + Self(CompletionInner::Direct { rx }) } +} - fn submit(&self, op: impl FnOnce(oneshot::Sender) -> Box) -> Completion { - let (tx, rx) = oneshot::channel(); - self.inner.lock().submit(op(tx)); - Completion(rx) - } +impl Future for Completion { + type Output = T; - fn submit_all>>( - &self, - ops: impl FnOnce(oneshot::Sender) -> I, - ) -> Completion { - let (tx, rx) = oneshot::channel(); - let mut inner = self.inner.lock(); - ops(tx).for_each(|op| inner.submit(op)); - Completion(rx) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + Pin::new(&mut this.0).poll(cx) } } -#[must_use = "completions must be polled to completion"] -pub struct Completion(oneshot::Receiver); +enum CompletionInner { + Direct { + rx: oneshot::Receiver, + }, + Mapped { + rx: oneshot::Receiver>>, + map: fn(Result>) -> T, + }, +} -impl Future for Completion { +impl Future for CompletionInner { type Output = T; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::new(&mut self.as_mut().0).poll(cx).map(Result::unwrap) + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match this { + Self::Direct { rx } => Pin::new(rx) + .poll(cx) + .map(|result| result.expect("lost completion sender")), + Self::Mapped { rx, map } => Pin::new(rx).poll(cx).map(|result| { + let result = result.expect("lost completion sender"); + map(result) + }), + } } } +enum CompletionHandle { + Write { + tx: oneshot::Sender>>, + buf_key: usize, + }, + Read { + tx: oneshot::Sender>>, + buf_key: usize, + }, + Open { + tx: oneshot::Sender>, + }, + Create { + tx: oneshot::Sender>, + }, + Stat { + tx: oneshot::Sender>, + }, + Fallocate { + tx: oneshot::Sender>, + }, + Fsync { + tx: oneshot::Sender>, + }, + Fdatasync { + tx: oneshot::Sender>, + }, + // TODO: We may use this for timeouts. + #[allow(unused)] + Noop { + tx: oneshot::Sender>, + }, +} + impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; type Completion = Completion; fn open_file(&self, path: &str) -> Self::Completion> { - self.submit(op::open_file(path)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::open(path).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Open { tx }); + } + } + + rx.into() } - fn create_file(&self, path: &str, len: u64) -> Self::Completion> { - self.submit(op::create_file(path, len)) + fn create_file(&self, path: &str) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::create(path).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Create { tx }); + } + } + + rx.into() } fn write_all_at( @@ -152,16 +297,29 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let () = B::ASSERT_VALID_LAYOUT; - - if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit(op::ready(Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }))) - } else { - self.submit_all(op::write_at(fd, buf, offset)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + let erased_buf = ErasedBox::from_aligned(buf); + let buf_ptr = erased_buf.as_ptr(); + + match executor.submit([Sqe::write(fd, buf_ptr, offset).attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: erased_buf, + })) + .unwrap_or_else(|_| unreachable!("rx is still alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(erased_buf); + pending_entry.insert(CompletionHandle::Write { tx, buf_key }); + } } + + Completion::mapped(rx, reify) } fn read_exact_at( @@ -170,106 +328,151 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let () = B::ASSERT_VALID_LAYOUT; - - if !offset.is_multiple_of(SECTOR_SIZE as _) { - self.submit(op::ready(Err(ErrorWith { - error: fs::Error::UnalignedOffset.into(), - with: buf, - }))) - } else { - self.submit_all(op::read_at(fd, buf, offset)) + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + let erased_buf = ErasedBox::from_aligned(buf); + let buf_ptr = erased_buf.as_ptr(); + + match executor.submit([Sqe::read(fd, buf_ptr, offset).attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: erased_buf, + })) + .unwrap_or_else(|_| unreachable!("rx is still alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(erased_buf); + pending_entry.insert(CompletionHandle::Read { tx, buf_key }); + } } - } - fn fsync(&self, _fd: Self::Fd) -> Self::Completion> { - self.submit(op::ready(Ok(()))) + Completion::mapped(rx, reify) } - fn fdatasync(&self, _fd: Self::Fd) -> Self::Completion> { - self.submit(op::ready(Ok(()))) - } + fn fsync(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); - fn reserve(&self, fd: Self::Fd, total: u64) -> Self::Completion> { - assert!(total >= fd.len()); - self.submit(op::set_len(fd, total)) - } + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::fsync(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fsync { tx }); + } + } - fn length(&self, fd: Self::Fd) -> Self::Completion> { - self.submit(op::get_len(fd)) + rx.into() } -} -#[derive(Default)] -struct SimulatorIOInner { - files: BTreeMap, fs::File>, - submissions: VecDeque>, - completions: VecDeque>, -} + fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); -impl SimulatorIOInner { - fn tick(&mut self) -> bool { - let mut progress = false; - if let Some(sqe) = self.submissions.pop_front() { - if let Some(cqe) = sqe.execute(&mut self.files) { - self.completions.push_back(cqe); + match executor.submit([Sqe::fdatasync(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fdatasync { tx }); } - progress = true; - } - if let Some(cqe) = self.completions.pop_front() { - cqe.complete(); - progress = true; } - progress + rx.into() } - fn execute(&mut self, sqe: Box) { - if let Some(cqe) = sqe.execute(&mut self.files) { - self.completions.push_back(cqe); + fn reserve(&self, fd: Self::Fd, total_size: u64) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([Sqe::fallocate(fd, total_size).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Fallocate { tx }); + } } - } - fn next_submission(&mut self) -> Option> { - self.submissions.pop_front() + rx.into() } - fn random_submission(&mut self, rng: &Rng) -> Option> { - let len = NonZeroUsize::new(self.submissions.len())?; - self.submissions.remove(rng.index(len.get())) - } + fn statx(&self, fd: Self::Fd) -> Self::Completion> { + let (tx, rx) = oneshot::channel(); - fn next_completion(&mut self) -> Option> { - self.completions.pop_front() - } + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); - fn random_completion(&mut self, rng: &Rng) -> Option> { - let len = NonZeroUsize::new(self.completions.len())?; - self.completions.remove(rng.index(len.get())) + match executor.submit([Sqe::stat(fd).attach(pending_entry.key())]) { + Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), + Ok(()) => { + pending_entry.insert(CompletionHandle::Stat { tx }); + } + } + + rx.into() } +} - fn submit(&mut self, op: Box) { - self.submissions.push_back(op); +fn reify( + result: Result>, +) -> Result> { + match result { + Ok(erased) => Ok(erased.into_aligned::()), + Err(ErrorWith { error, with }) => Err(ErrorWith { + error, + with: with.into_aligned::(), + }), } } #[cfg(test)] mod tests { - use crate::sim::{Runtime, RuntimeConfig}; + use crate::sim::{time::TimeHandle, GlobalRng}; use super::*; + struct Runtime { + rt: tokio::runtime::LocalRuntime, + io: SimulatorIO, + rng: Rng, + time: TimeHandle, + } + + impl Runtime { + fn new() -> Self { + Self { + rt: tokio::runtime::Builder::new_current_thread() + .build_local(<_>::default()) + .unwrap(), + io: SimulatorIO::default(), + rng: GlobalRng::new(0), + time: TimeHandle::default(), + } + } + + fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { + let fut = self.rt.spawn_local(f(&self.io)); + while self.io.tick(&self.rng, self.time.now()) {} + self.rt.block_on(fut).unwrap() + } + } + #[test] fn create_file() { - let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); - let io = rt.io().cloned().unwrap(); - - let fd = rt - .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) - .unwrap(); - assert_eq!(fd.len(), 2 * SECTOR_SIZE as u64); + let rt = Runtime::new(); + rt.run(|io| io.create_file("/data/test")).unwrap(); } + #[derive(Debug)] #[repr(C, align(4096))] struct Buf([u8; 2 * SECTOR_SIZE]); @@ -298,21 +501,14 @@ mod tests { #[test] fn write_read_roundtrip() { - let mut rt = Runtime::with_config(RuntimeConfig::default().enable_io()); - let io = rt.io().cloned().unwrap(); + let rt = Runtime::new(); - let fd = rt - .block_on(io.create_file("/data/test", 2 * SECTOR_SIZE as u64)) - .unwrap(); + let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); let mut buf = rt - .block_on(io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) - .map_err(|ErrorWith { error, .. }| error) + .run(|io| io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) .unwrap(); buf.clear(); - let buf = rt - .block_on(io.read_exact_at(fd, buf, 0)) - .map_err(|ErrorWith { error, .. }| error) - .unwrap(); + let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); assert!(buf.0.iter().all(|&b| b == 22)); } diff --git a/crates/runtime-core/src/sim/io/op.rs b/crates/runtime-core/src/sim/io/op.rs deleted file mode 100644 index 0e855313819..00000000000 --- a/crates/runtime-core/src/sim/io/op.rs +++ /dev/null @@ -1,563 +0,0 @@ -use core::{any::Any, iter::Scan, ops::Range}; - -use alloc::{ - boxed::Box, - collections::{btree_map, BTreeMap}, - sync::Arc, -}; -use futures_channel::oneshot; - -use super::{fs, Error}; -use crate::io::{AlignedBytes, ErrorWith, SECTOR_SIZE}; - -/// An operation that can be submitted to the [super::SimulatorIO] driver. -pub trait Submission: Send + Any { - /// Run the operations with mutable access to the currently registered - /// [fs::File]s. - /// - /// If the operation is done, a [Completion] is returned in a `Some`. - /// `None` may be returned if: - /// - /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. - /// - The submission is a [Noop]. - /// - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option>; - - /// Cancel the operation instead of executing it. - /// - /// This will generate a [Completion] with the result [Error::Cancelled], - /// unless: - /// - /// - The submission is a sub-operation, such as [WritePage] or [ReadPage]. - /// - The submission is a [Noop]. - /// - fn cancel(self: Box) -> Option>; -} - -/// An object containing the result of executing a [Submission], as well as a -/// handle to resolve a future waiting on the outcome of the operation. -pub trait Completion: Send { - fn success(&self) -> bool; - /// Resolve the future waiting on the outcome of the operation. - fn complete(self: Box); -} - -/// A channel to resolve a future waiting on the outcome of a submitted -/// operation. -pub type OnComplete = oneshot::Sender; - -pub type ScanState = (fs::File, usize, Arc>>); -pub type PageWrites = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; -pub type PageReads = Scan, ScanState, fn(&mut ScanState, usize) -> Option>>; - -pub type WriteAtResult = Result>; - -/// Write the contents of `buf` to `fd` at `offset`. -/// -/// This operation is split into multiple writes to individual pages. The -/// `on_complete` future resolves only after all page writes completed. -/// -/// Ownership of `buf` is transferred back when the operation completes. -pub fn write_at( - fd: fs::File, - buf: B, - offset: u64, -) -> impl FnOnce(OnComplete>) -> PageWrites { - move |on_complete| { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { - let op = WritePage { - fd: fd.clone(), - file_page: *first_page + buf_page, - buf_page, - state: state.clone(), - }; - - Some(Box::new(op)) - }) - } -} - -pub type ReadAtResult = Result>; - -/// Fill `buf` by reading from `fd` at `offset`. -/// -/// This operation is split into multple reads from the individual pages needed -/// to fill `buf`. The `on_complete` future resolves only after all page reads -/// completed. -/// -/// Ownership of `buf` is transferred back when the operation completes. -pub fn read_at( - fd: fs::File, - buf: B, - offset: u64, -) -> impl FnOnce(OnComplete>) -> PageReads { - move |on_complete| { - let first_page = (offset / SECTOR_SIZE as u64) as usize; - let page_count = buf.as_bytes().len() / SECTOR_SIZE; - - let state = Arc::new(spin::Mutex::new(PagedOpState { - buf: Some(buf), - on_complete: Some(on_complete), - remaining: page_count, - first_error: None, - })); - - (0..page_count).scan((fd, first_page, state), |(fd, first_page, state), buf_page| { - let op = ReadPage { - fd: fd.clone(), - file_page: *first_page + buf_page, - buf_page, - state: state.clone(), - }; - - Some(Box::new(op)) - }) - } -} - -/// Open file at `path`. -pub fn open_file(path: &str) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| { - Box::new(OpenFile { - path: path.into(), - on_complete, - }) - } -} - -/// Create a new file at `path` and allocate `len` space for it. -pub fn create_file(path: &str, len: u64) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| { - Box::new(CreateFile { - path: path.into(), - len, - on_complete, - }) - } -} - -/// Get the length of the file `fd`. -pub fn get_len(fd: fs::File) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| Box::new(GetLen { fd, on_complete }) -} - -/// Set the length of the file `fd`. -pub fn set_len(fd: fs::File, len: u64) -> impl FnOnce(OnComplete>) -> Box { - move |on_complete| Box::new(SetLen { fd, len, on_complete }) -} - -struct GenericCompletion { - success: bool, - result: T, - on_complete: OnComplete, -} - -fn completion(success: bool, result: T, on_complete: OnComplete) -> Box { - Box::new(GenericCompletion { - success, - result, - on_complete, - }) -} - -impl Completion for GenericCompletion { - fn success(&self) -> bool { - self.success - } - - fn complete(self: Box) { - let Self { - result, on_complete, .. - } = *self; - let _ = on_complete.send(result); - } -} - -/// [Submission] created by [noop]. -pub(crate) struct Noop; - -impl Submission for Noop { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - None - } - - fn cancel(self: Box) -> Option> { - None - } -} - -/// An operation that does nothing. -/// -/// Note that no completion is associated with a noop, but the submission still -/// occupies a slot in the submission queue. -pub fn noop() -> Box { - Box::new(Noop) -} - -/// [Submission] created by [ready]. -pub(crate) struct Ready(Box); - -impl Submission for Ready { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self(completion) = *self; - Some(completion) - } - - fn cancel(self: Box) -> Option> { - let Self(completion) = *self; - Some(completion) - } -} - -/// An operation that is already complete with `result`. -pub fn ready(result: T) -> impl FnOnce(OnComplete) -> Box { - move |on_complete| Box::new(Ready(completion(true, result, on_complete))) -} - -/// [Submission] created by [link]. -pub(crate) struct SoftLink { - a: Box, - b: Box, -} - -impl Submission for SoftLink { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { a, b } = *self; - let result_a = a.execute(files); - let result_b = if result_a.as_ref().is_none_or(|result| result.success()) { - b.execute(files) - } else { - b.cancel() - }; - - Some(Box::new(LinkedCompletion { - a: result_a, - b: result_b, - })) - } - - fn cancel(self: Box) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.cancel(), - b: b.cancel(), - })) - } -} - -/// Link `a` and `b`, such that `b` gets executed after `a`. -/// -/// If `a` fails (i.e. its [Completion::success] returns `false`), `b` is -/// cancelled. -/// -/// Corresponds to io-uring's `IOSQE_IO_LINK` flag. To emulate -/// `IOSQE_IO_HARDLINK`, see [hard_link]. -pub fn link(a: Box, b: Box) -> Box { - Box::new(SoftLink { a, b }) -} - -/// [Submission] created by [hard_link]. -pub(crate) struct HardLink { - a: Box, - b: Box, -} - -impl Submission for HardLink { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.execute(files), - b: b.execute(files), - })) - } - - fn cancel(self: Box) -> Option> { - let Self { a, b } = *self; - Some(Box::new(LinkedCompletion { - a: a.cancel(), - b: b.cancel(), - })) - } -} - -/// Link `a` and `b`, such that `b` gets executed after `a`. -/// -/// Unlike [link], this executes both submissions regardless of the result. It -/// just enforces the ordering constraint that `b` will never execute before -/// `a`. -/// -/// Corresponds to io-uring's `IOSQE_IO_HARDLINK` flag. To emulate -/// `IOSQE_IO_LINK`, see [link]. -pub fn hard_link(a: Box, b: Box) -> Box { - Box::new(HardLink { a, b }) -} - -struct LinkedCompletion { - a: Option>, - b: Option>, -} - -impl Completion for LinkedCompletion { - fn success(&self) -> bool { - self.a.as_ref().is_none_or(|result| result.success()) && self.b.as_ref().is_none_or(|result| result.success()) - } - - fn complete(self: Box) { - let Self { a, b } = *self; - if let Some(a) = a { - a.complete(); - } - if let Some(b) = b { - b.complete(); - } - } -} - -/// [Submission] created by [open_file]. -pub(crate) struct OpenFile { - path: Box, - on_complete: OnComplete>, -} - -impl Submission for OpenFile { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { path, on_complete } = *self; - let result = files.get(&path).cloned().ok_or(Error::FileNotFound { path }); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { path: _, on_complete } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [create_file]. -pub(crate) struct CreateFile { - path: Box, - len: u64, - on_complete: OnComplete>, -} - -impl Submission for CreateFile { - fn execute(self: Box, files: &mut BTreeMap, fs::File>) -> Option> { - let Self { path, len, on_complete } = *self; - let result = (|| { - let file = match files.entry(path.clone()) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(_) => Err(Error::FileAlreadyExists { path }), - }?; - file.set_len(len)?; - Ok(file) - })(); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { - path: _, - len: _, - on_complete, - } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [get_len]. -pub(crate) struct GetLen { - fd: fs::File, - on_complete: OnComplete>, -} - -impl Submission for GetLen { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { fd, on_complete } = *self; - let result = Ok(fd.len()); - Some(completion(true, result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { fd: _, on_complete } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -/// [Submission] created by [set_len]. -pub(crate) struct SetLen { - fd: fs::File, - len: u64, - on_complete: OnComplete>, -} - -impl Submission for SetLen { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { fd, len, on_complete } = *self; - let result = fd.set_len(len).map_err(Error::from); - Some(completion(result.is_ok(), result, on_complete)) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - len: _, - on_complete, - } = *self; - Some(completion(false, Err(Error::Cancelled), on_complete)) - } -} - -pub struct PagedOpState { - buf: Option, - on_complete: Option>>>, - remaining: usize, - first_error: Option, -} - -fn complete_page_op( - state: &Arc>>, - result: Result<(), Error>, -) -> Option>> { - let complete = { - let mut state = state.lock(); - if let Err(e) = result - && state.first_error.is_none() - { - state.first_error.replace(e); - } - assert!(state.remaining > 0); - state.remaining -= 1; - - state.remaining == 0 - }; - - complete.then(|| Box::new(PageOpCompletion { state: state.clone() })) -} - -struct PageOpCompletion { - state: Arc>>, -} - -impl Completion for PageOpCompletion { - fn success(&self) -> bool { - self.state.lock().first_error.is_none() - } - - fn complete(self: Box) { - let (on_complete, result) = { - let mut state = self.state.lock(); - - assert_eq!(state.remaining, 0); - - let buf = state.buf.take().expect("write completed more than once"); - let on_complete = state.on_complete.take().expect("write completed more than once"); - - let result = match state.first_error.take() { - None => Ok(buf), - Some(error) => Err(ErrorWith { error, with: buf }), - }; - - (on_complete, result) - }; - - let _ = on_complete.send(result); - } -} - -pub(crate) struct WritePage { - fd: fs::File, - file_page: usize, - buf_page: usize, - state: Arc>>, -} - -impl Submission for WritePage { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { - fd, - file_page, - buf_page, - state, - } = *self; - - let result = { - let state_ref = state.lock(); - let buf = state_ref.buf.as_ref().expect("buffer went away"); - - let start = buf_page * SECTOR_SIZE; - let end = start + SECTOR_SIZE; - fd.write_page(&buf.as_bytes()[start..end], file_page as _) - }; - complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - file_page: _, - buf_page: _, - state, - } = *self; - complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) - } -} - -pub(crate) struct ReadPage { - fd: fs::File, - file_page: usize, - buf_page: usize, - state: Arc>>, -} - -impl Submission for ReadPage { - fn execute(self: Box, _files: &mut BTreeMap, fs::File>) -> Option> { - let Self { - fd, - file_page, - buf_page, - state, - } = *self; - - let result = { - let mut state_ref = state.lock(); - let buf = state_ref.buf.as_mut().expect("buffer went away"); - - let start = buf_page * SECTOR_SIZE; - let end = start + SECTOR_SIZE; - fd.read_page(&mut buf.as_bytes_mut()[start..end], file_page as _) - }; - complete_page_op(&state, result.map_err(Into::into)).map(|c| c as Box) - } - - fn cancel(self: Box) -> Option> { - let Self { - fd: _, - file_page: _, - buf_page: _, - state, - } = *self; - complete_page_op(&state, Err(Error::Cancelled)).map(|c| c as Box) - } -} - -#[cfg(test)] -mod tests { - use core::any::Any; - - use super::*; - - #[test] - fn downcast() { - let sqe: Box = noop(); - sqe.downcast::().unwrap(); - } -} diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 1aeb31c0af1..c49f9f690b6 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,7 +5,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO}; +use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO, Statx}; use static_assertions::assert_not_impl_any; use tokio::runtime; @@ -80,15 +80,13 @@ impl SpacetimeIO for TokioIO { .into() } - fn create_file(&self, path: &str, len: u64) -> Self::Completion> { + fn create_file(&self, path: &str) -> Self::Completion> { let path = PathBuf::from(path); self.rt .spawn_blocking(move || { let mut open_options = std::fs::File::options(); open_options.read(true).write(true).create_new(true); - let file = platform::open_with_direct_io(open_options, path)?; - file.set_len(len)?; - Ok(Arc::new(file)) + platform::open_with_direct_io(open_options, path).map(Arc::new) }) .into() } @@ -140,11 +138,11 @@ impl SpacetimeIO for TokioIO { .into() } - fn length(&self, fd: Self::Fd) -> Self::Completion> { + fn statx(&self, fd: Self::Fd) -> Self::Completion> { self.rt .spawn_blocking(move || { let mut fd = fd.try_clone()?; - file_length(&mut fd) + file_length(&mut fd).map(Statx::from_size) }) .into() } From 059a6ac5454047bde10d1068aa679ff64ebbffef Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 10:06:01 +0200 Subject: [PATCH 14/47] Adjust queuing / overflow behavior to match io-uring more closely --- crates/runtime-core/src/sim/io/executor.rs | 156 +++++++++++++++------ crates/runtime-core/src/sim/io/mod.rs | 4 +- 2 files changed, 116 insertions(+), 44 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index 9135fa45c7c..a4f6eb2af70 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -5,7 +5,8 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::result::Result; +use core::{num::NonZeroUsize, result::Result}; +use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, @@ -15,9 +16,18 @@ use crate::{ }, }; +/// Dependency on the previous [Sqe]. +/// +/// A link imposes an ordering constraint: the [Sqe] carrying the link will not +/// be executed before the preceding one completed. Note that a link is only +/// meaningful within a batch of SQEs submitted together. #[derive(Clone, Copy)] pub enum LinkKind { + /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. + /// Analogous to `IOSQE_IO_LINK`. Soft, + /// Run the SQE regardless of the preceding SQE's result. + /// Analoguous to `IOSQE_IO_HARDLINK`. Hard, } @@ -447,6 +457,7 @@ enum InFlightInner { Noop, } +/* pub enum WriteFault { /// Misdirect the write to an arbitrary page offset in the file. Misdirected { page_offset: usize }, @@ -465,25 +476,86 @@ pub enum WriteFault { pub trait FaultInjector { fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; } +*/ + +/// Completion queue overflow policy. +/// +/// Note that we do **not** model `IORING_FEAT_NODROP`, because we never want +/// the application to rely on dynamic memory allocation in the kernel. +/// +/// The default is to panic, which should prompt the user to adjust queue size +/// configuration. However, sometimes it may be useful to see how the +/// application behaves when completions are dropped. +#[derive(Default)] +pub enum OnCqOverflow { + #[default] + Panic, + Drop, +} + +pub struct Options { + /// Capacity of the submission queue. + /// + /// This basically limits how many [Sqe]s can be submitted in one batch. + /// Should be a power of 2, or is otherwise rounded up to the next power of + /// 2. + pub capacity: NonZeroUsize, + /// Override the completion queue capacity. + /// + /// By default, the completion queue's capacity is twice the submission + /// queue's. This can be insufficient for some workloads, so this setting + /// can be used to override the default. + /// + /// Should be a power of 2, or is otherwise rounder up to the next power of + /// two. + pub cq_capacity: Option, + /// What to do if the completion queue overflows. + pub cq_overflow: OnCqOverflow, +} + +impl Default for Options { + fn default() -> Self { + Self { + capacity: NonZeroUsize::new(8).unwrap(), + cq_capacity: None, + cq_overflow: OnCqOverflow::default(), + } + } +} -pub struct Executor { +pub struct Executor { submissions: VecDeque>, completions: VecDeque>, - in_flight: [Option>; MAX_INFLIGHT], + in_flight: Slab>, executing: VecDeque, fstree: BTreeMap, fs::File>, -} -impl Executor { - pub fn with_capacity(capacity: usize) -> Self { + cq_overflow: OnCqOverflow, + cq_dropped: usize, +} + +impl Executor { + pub fn new( + Options { + capacity, + cq_capacity, + cq_overflow, + }: Options, + ) -> Self { + let sq_capacity = capacity.get().next_power_of_two(); + let cq_capacity = cq_capacity + .map(|c| c.get().next_power_of_two()) + .unwrap_or_else(|| 2 * sq_capacity); Self { - submissions: VecDeque::with_capacity(capacity), - completions: VecDeque::with_capacity(capacity), - in_flight: core::array::from_fn(|_| None), + submissions: VecDeque::with_capacity(sq_capacity), + completions: VecDeque::with_capacity(cq_capacity), + in_flight: Slab::new(), executing: VecDeque::new(), fstree: BTreeMap::new(), + cq_overflow, + cq_dropped: 0, } } @@ -502,13 +574,22 @@ impl Executor { } fn complete(&mut self, cqe: Cqe) { - assert!( - self.completions.len() < self.completions.capacity(), - "completion queue overflow" - ); + if self.completions.len() == self.completions.capacity() { + match self.cq_overflow { + OnCqOverflow::Panic => panic!("completion queue overflow"), + OnCqOverflow::Drop => { + self.cq_dropped += 1; + return; + } + } + } self.completions.push_back(cqe); } + pub fn dropped_completions(&self) -> usize { + self.cq_dropped + } + pub fn completed(&mut self) -> impl Iterator> { self.completions.drain(..) } @@ -522,12 +603,7 @@ impl Executor { fn schedule(&mut self) -> bool { let mut progress = false; - // Fill free execution slots. - for (id, slot) in self.in_flight.iter_mut().filter(|f| f.is_none()).enumerate() { - let Some(sqe) = self.submissions.pop_front() else { - break; - }; - + while let Some(sqe) = self.submissions.pop_front() { // If the sqe is linked, pop the whole chain. // Links of sqes not submitted in the same batch are ignored. let mut successors = VecDeque::new(); @@ -545,10 +621,10 @@ impl Executor { } } } - - let (in_flight, ops) = sqe.inner.schedule(id); + let slot = self.in_flight.vacant_entry(); + let (in_flight, ops) = sqe.inner.schedule(slot.key()); self.executing.extend(ops); - slot.replace(InFlight { + slot.insert(InFlight { inner: in_flight, blocked: successors, user_data: sqe.user_data, @@ -580,7 +656,7 @@ impl Executor { results, }, .. - } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected write") }; @@ -604,7 +680,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected write") }; @@ -634,7 +710,7 @@ impl Executor { results, }, .. - } = self.in_flight[sqe].as_mut().expect("invalid sqe id") + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected read") }; @@ -658,7 +734,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invlid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected read") }; @@ -679,7 +755,7 @@ impl Executor { inner: InFlightInner::Open { sqe: Open { path } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected open") }; @@ -694,7 +770,7 @@ impl Executor { inner: InFlightInner::Create { sqe: Create { path } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected create") }; @@ -714,7 +790,7 @@ impl Executor { inner: InFlightInner::Stat { sqe: Stat { fd } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected stat") }; @@ -733,7 +809,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fallocate") }; @@ -750,7 +826,7 @@ impl Executor { inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fsync") }; @@ -770,7 +846,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected fdatasync") }; @@ -786,7 +862,7 @@ impl Executor { inner: InFlightInner::Noop, blocked, user_data, - } = self.in_flight[sqe].take().expect("invalid sqe id") + } = self.in_flight.remove(sqe) else { unreachable!("invalid sqe: expected noop") }; @@ -805,12 +881,7 @@ impl Executor { false } - fn schedule_linked( - &mut self, - in_flight_slot: SqeId, - prev_succeeded: bool, - mut blocked: VecDeque>, - ) { + fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { if let Some(Blocked { link, sqe: next, @@ -830,13 +901,14 @@ impl Executor { } } (LinkKind::Soft, true) | (LinkKind::Hard, _) => { - let (inner, ops) = next.schedule(in_flight_slot); + let (inner, ops) = next.schedule(sqe); self.executing.extend(ops); - self.in_flight[in_flight_slot].replace(InFlight { + let slot = self.in_flight.get_mut(sqe).expect("invalid sqe id"); + *slot = InFlight { inner, blocked, user_data, - }); + }; } } } diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index d981ba38193..831dcacb08d 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -149,7 +149,7 @@ impl SimulatorIO { } struct SimulatorInner { - executor: spin::Mutex>, + executor: spin::Mutex>, pending: spin::Mutex>, buffers: Arc>>, } @@ -157,7 +157,7 @@ struct SimulatorInner { impl Default for SimulatorInner { fn default() -> Self { Self { - executor: spin::Mutex::new(Executor::with_capacity(128)), + executor: spin::Mutex::new(Executor::new(<_>::default())), pending: <_>::default(), buffers: <_>::default(), } From 31baac575417fdca32ec0abc229f74874d98d51e Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 11:13:29 +0200 Subject: [PATCH 15/47] Slight cleanup --- crates/runtime-core/src/sim/io/executor.rs | 2 - crates/runtime-core/src/sim/io/mod.rs | 214 ++++++++------------- 2 files changed, 77 insertions(+), 139 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index a4f6eb2af70..a9772833bef 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -457,7 +457,6 @@ enum InFlightInner { Noop, } -/* pub enum WriteFault { /// Misdirect the write to an arbitrary page offset in the file. Misdirected { page_offset: usize }, @@ -476,7 +475,6 @@ pub enum WriteFault { pub trait FaultInjector { fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; } -*/ /// Completion queue overflow policy. /// diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 831dcacb08d..f887ad6cf3e 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -146,6 +146,57 @@ impl SimulatorIO { progress } + + fn submit( + &self, + sqe: Sqe, + completion_handle: impl FnOnce(CompletionSender) -> CompletionHandle, + ) -> Completion> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([sqe.attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(Error::SubmissionQueueOverflow)) + .unwrap_or_else(|_| unreachable!("rx is alive")), + Ok(()) => { + pending_entry.insert(completion_handle(tx)); + } + } + + rx.into() + } + + fn submit_with( + &self, + sqe: Sqe, + buf: ErasedBox, + completion_handle: impl FnOnce(CompletionSender>, usize) -> CompletionHandle, + ) -> Completion>> { + let (tx, rx) = oneshot::channel(); + + let mut executor = self.inner.executor.lock(); + let mut pending = self.inner.pending.lock(); + let pending_entry = pending.vacant_entry(); + + match executor.submit([sqe.attach(pending_entry.key())]) { + Err(_sqe) => tx + .send(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: buf, + })) + .unwrap_or_else(|_| unreachable!("rx is alive")), + Ok(()) => { + let buf_key = self.inner.buffers.lock().insert(buf); + pending_entry.insert(completion_handle(tx, buf_key)); + } + } + + Completion::mapped(rx, reify) + } } struct SimulatorInner { @@ -164,12 +215,14 @@ impl Default for SimulatorInner { } } +pub type CompletionReceiver = oneshot::Receiver>; + #[must_use = "completions must be polled to completion"] pub struct Completion(CompletionInner); impl Completion { pub fn mapped( - rx: oneshot::Receiver>>, + rx: CompletionReceiver>, map: fn(Result>) -> T, ) -> Self { Self(CompletionInner::Mapped { rx, map }) @@ -196,7 +249,7 @@ enum CompletionInner { rx: oneshot::Receiver, }, Mapped { - rx: oneshot::Receiver>>, + rx: CompletionReceiver>, map: fn(Result>) -> T, }, } @@ -218,37 +271,38 @@ impl Future for CompletionInner { } } +type CompletionSender = oneshot::Sender>; enum CompletionHandle { Write { - tx: oneshot::Sender>>, + tx: CompletionSender>, buf_key: usize, }, Read { - tx: oneshot::Sender>>, + tx: CompletionSender>, buf_key: usize, }, Open { - tx: oneshot::Sender>, + tx: CompletionSender, }, Create { - tx: oneshot::Sender>, + tx: CompletionSender, }, Stat { - tx: oneshot::Sender>, + tx: CompletionSender, }, Fallocate { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, Fsync { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, Fdatasync { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, // TODO: We may use this for timeouts. #[allow(unused)] Noop { - tx: oneshot::Sender>, + tx: CompletionSender<(), Error>, }, } @@ -258,37 +312,11 @@ impl SpacetimeIO for SimulatorIO { type Completion = Completion; fn open_file(&self, path: &str) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::open(path).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Open { tx }); - } - } - - rx.into() + self.submit(Sqe::open(path), |tx| CompletionHandle::Open { tx }) } fn create_file(&self, path: &str) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::create(path).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Create { tx }); - } - } - - rx.into() + self.submit(Sqe::create(path), |tx| CompletionHandle::Create { tx }) } fn write_all_at( @@ -297,29 +325,11 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - let erased_buf = ErasedBox::from_aligned(buf); let buf_ptr = erased_buf.as_ptr(); - - match executor.submit([Sqe::write(fd, buf_ptr, offset).attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: erased_buf, - })) - .unwrap_or_else(|_| unreachable!("rx is still alive")), - Ok(()) => { - let buf_key = self.inner.buffers.lock().insert(erased_buf); - pending_entry.insert(CompletionHandle::Write { tx, buf_key }); - } - } - - Completion::mapped(rx, reify) + self.submit_with(Sqe::write(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { + CompletionHandle::Write { tx, buf_key } + }) } fn read_exact_at( @@ -328,97 +338,27 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - let erased_buf = ErasedBox::from_aligned(buf); let buf_ptr = erased_buf.as_ptr(); - - match executor.submit([Sqe::read(fd, buf_ptr, offset).attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: erased_buf, - })) - .unwrap_or_else(|_| unreachable!("rx is still alive")), - Ok(()) => { - let buf_key = self.inner.buffers.lock().insert(erased_buf); - pending_entry.insert(CompletionHandle::Read { tx, buf_key }); - } - } - - Completion::mapped(rx, reify) + self.submit_with(Sqe::read(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { + CompletionHandle::Read { tx, buf_key } + }) } fn fsync(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fsync(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fsync { tx }); - } - } - - rx.into() + self.submit(Sqe::fsync(fd), |tx| CompletionHandle::Fsync { tx }) } fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fdatasync(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fdatasync { tx }); - } - } - - rx.into() + self.submit(Sqe::fdatasync(fd), |tx| CompletionHandle::Fdatasync { tx }) } fn reserve(&self, fd: Self::Fd, total_size: u64) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::fallocate(fd, total_size).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Fallocate { tx }); - } - } - - rx.into() + self.submit(Sqe::fallocate(fd, total_size), |tx| CompletionHandle::Fallocate { tx }) } fn statx(&self, fd: Self::Fd) -> Self::Completion> { - let (tx, rx) = oneshot::channel(); - - let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - match executor.submit([Sqe::stat(fd).attach(pending_entry.key())]) { - Err(_sqe) => tx.send(Err(Error::SubmissionQueueOverflow)).unwrap(), - Ok(()) => { - pending_entry.insert(CompletionHandle::Stat { tx }); - } - } - - rx.into() + self.submit(Sqe::stat(fd), |tx| CompletionHandle::Stat { tx }) } } From 80f2787521ea7c3568db149a554afb2a72cb7e5e Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 2 Sep 2026 14:54:13 +0200 Subject: [PATCH 16/47] Model file durability --- crates/runtime-core/src/sim/io/executor.rs | 500 +++++++++++++-------- crates/runtime-core/src/sim/io/fs.rs | 152 +++++-- 2 files changed, 422 insertions(+), 230 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index a9772833bef..d2e34404dc0 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -5,13 +5,16 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::{num::NonZeroUsize, result::Result}; +use core::{mem, num::NonZeroUsize, result::Result}; use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, sim::{ - io::{fs, Error, Instant}, + io::{ + fs::{self, Datasync}, + Error, Instant, + }, Rng, }, }; @@ -221,14 +224,48 @@ impl SqeInner { InFlightInner::Fallocate { sqe }, alloc::vec![Operation::Fallocate { sqe: sqe_id }], ), - SqeInner::Fsync(sqe) => ( - InFlightInner::Fsync { sqe }, - alloc::vec![Operation::Fsync { sqe: sqe_id }], - ), - SqeInner::Fdatasync(sqe) => ( - InFlightInner::Fdatasync { sqe }, - alloc::vec![Operation::Fdatasync { sqe: sqe_id }], - ), + SqeInner::Fsync(sqe) => { + let Fsync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Sector(offset), + }) + .chain([Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Length, + }]) + .collect::>(); + let in_flight = InFlightInner::Fsync { + sqe, + op_count: ops.len(), + }; + + (in_flight, ops) + } + SqeInner::Fdatasync(sqe) => { + let Fdatasync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Sector(offset), + }) + .chain([Operation::Fdatasync { + sqe: sqe_id, + effect: Datasync::Length, + }]) + .collect::>(); + let in_flight = InFlightInner::Fdatasync { + sqe, + op_count: ops.len(), + }; + + (in_flight, ops) + } SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), } } @@ -379,6 +416,12 @@ impl Cqe { type SqeId = usize; +// TODO: There is no difference between fsync and fdatasync as long as we don't +// have an API to fsync the directory of a file after it was created. +enum FsyncEffect { + Datasync(Datasync), +} + enum Operation { WriteSector { sqe: SqeId, @@ -404,9 +447,11 @@ enum Operation { }, Fsync { sqe: SqeId, + effect: FsyncEffect, }, Fdatasync { sqe: SqeId, + effect: Datasync, }, Noop { sqe: SqeId, @@ -450,9 +495,11 @@ enum InFlightInner { }, Fsync { sqe: Fsync, + op_count: usize, }, Fdatasync { sqe: Fdatasync, + op_count: usize, }, Noop, } @@ -484,7 +531,7 @@ pub trait FaultInjector { /// The default is to panic, which should prompt the user to adjust queue size /// configuration. However, sometimes it may be useful to see how the /// application behaves when completions are dropped. -#[derive(Default)] +#[derive(Clone, Copy, Default)] pub enum OnCqOverflow { #[default] Panic, @@ -557,6 +604,43 @@ impl Executor { } } + /// Simulate a power-loss crash. + /// + /// All submitted and executing operations are cancelled, and files reset to + /// their durable state. After this method returns, the completion queue is + /// empty. + pub fn crash(&mut self) { + self.submissions.clear(); + self.completions.clear(); + self.in_flight.clear(); + self.executing.clear(); + self.cq_dropped = 0; + + for file in self.fstree.values_mut() { + file.crash(); + } + } + + /// Restart the executor, simulating a process crash. + /// + /// Unlike [Self::crash], this will drive the currently executing operations + /// to completion. Submissions that were not yet scheduled are dropped. The + /// file state remains unchanged. + /// + /// After this method returns, the completion queue is empty. + pub fn restart(&mut self, now: Instant) { + self.submissions.clear(); + let cq_overflow_orig = self.cq_overflow; + self.cq_overflow = OnCqOverflow::Drop; + let executing = mem::take(&mut self.executing); + for op in executing { + self.execute(op, now); + } + self.completions.clear(); + self.cq_overflow = cq_overflow_orig; + self.cq_dropped = 0; + } + pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> where Batch: IntoIterator>, @@ -594,7 +678,7 @@ impl Executor { pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { let mut progress = self.schedule(); - progress |= self.execute(rng, now); + progress |= self.execute_random(rng, now); progress } @@ -634,249 +718,275 @@ impl Executor { progress } - fn execute(&mut self, rng: &Rng, now: Instant) -> bool { + fn execute_random(&mut self, rng: &Rng, now: Instant) -> bool { if self.executing.is_empty() { return false; } if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { - match op { - Operation::WriteSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected write") - }; - let bytes = buf.as_bytes(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &buf.as_bytes()[buf_offset..end]; - let result = fd.write_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected write") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Write { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::ReadSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected read") - }; - let bytes = buf.as_bytes_mut(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &mut buf.as_bytes_mut()[buf_offset..end]; - let result = fd.read_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; + self.execute(op, now); + true + } else { + false + } + } - if is_complete { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected read") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Read { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::Open { sqe } => { + fn execute(&mut self, op: Operation, _now: Instant) { + match op { + Operation::WriteSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { let InFlight { - inner: InFlightInner::Open { sqe: Open { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Write { + sqe: Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { - unreachable!("invalid sqe: expected open") + unreachable!("invalid sqe: expected write") }; + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); - let is_success = result.is_ok(); - self.complete(Cqe::Open { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Create { sqe } => { + let buf = &buf.as_bytes()[buf_offset..end]; + let result = fd.write_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { let InFlight { - inner: InFlightInner::Create { sqe: Create { path } }, + inner: + InFlightInner::Write { + sqe: Write { mut buf, .. }, + op_count, + results, + }, blocked, user_data, } = self.in_flight.remove(sqe) else { - unreachable!("invalid sqe: expected create") + unreachable!("invalid sqe: expected write") }; - - let result = match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { - path: entry.key().clone(), - }), + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), }; let is_success = result.is_ok(); - self.complete(Cqe::Create { result, user_data }); + self.complete(Cqe::Write { result, user_data }); self.schedule_linked(sqe, is_success, blocked); } - Operation::Stat { sqe } => { + } + Operation::ReadSector { + sqe, + page_offset, + buf_offset, + } => { + let is_complete = { let InFlight { - inner: InFlightInner::Stat { sqe: Stat { fd } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Read { + sqe: Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { - unreachable!("invalid sqe: expected stat") + unreachable!("invalid sqe: expected read") }; + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - self.complete(Cqe::Stat { - result: Ok(Statx { size: fd.len() }), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Fallocate { sqe } => { + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + let result = fd.read_page(buf, page_offset as _); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { let InFlight { inner: - InFlightInner::Fallocate { - sqe: Fallocate { fd, total_len }, + InFlightInner::Read { + sqe: Read { mut buf, .. }, + op_count, + results, }, blocked, user_data, } = self.in_flight.remove(sqe) else { - unreachable!("invalid sqe: expected fallocate") + unreachable!("invalid sqe: expected read") }; - - self.complete(Cqe::Fallocate { - result: fd.set_len(total_len).map_err(Error::from), - user_data, - }); - self.schedule_linked(sqe, true, blocked); + assert!(results.len() == op_count); + // TODO: Propagate all errors? + // TODO: Allow write op failures and reflect in returned number. + let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { + Some(error) => Err(error), + None => Ok(buf.as_bytes().len()), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); } - Operation::Fsync { sqe } => { - // TODO: Do something fallible with fd. + } + Operation::Open { sqe } => { + let InFlight { + inner: InFlightInner::Open { sqe: Open { path } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected open") + }; + + let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Create { sqe } => { + let InFlight { + inner: InFlightInner::Create { sqe: Create { path } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected create") + }; + + let result = match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + Operation::Stat { sqe } => { + let InFlight { + inner: InFlightInner::Stat { sqe: Stat { fd } }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected stat") + }; + + self.complete(Cqe::Stat { + result: Ok(Statx { size: fd.len() }), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fallocate { sqe } => { + let InFlight { + inner: + InFlightInner::Fallocate { + sqe: Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected fallocate") + }; + + self.complete(Cqe::Fallocate { + result: fd.set_len(total_len).map_err(Error::from), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } + Operation::Fsync { sqe, effect } => { + let is_complete = { let InFlight { - inner: InFlightInner::Fsync { sqe: Fsync { fd: _ } }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + inner: + InFlightInner::Fsync { + sqe: Fsync { fd }, + op_count, + }, + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fsync") }; + match effect { + FsyncEffect::Datasync(effect) => fd.fdatasync([effect]), + } + *op_count -= 1; + + *op_count == 0 + }; + + if is_complete { + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); self.complete(Cqe::Fsync { result: Ok(()), user_data, }); self.schedule_linked(sqe, true, blocked); } - Operation::Fdatasync { sqe } => { - // TODO: Do something fallible with fd. + } + Operation::Fdatasync { sqe, effect } => { + let is_complete = { let InFlight { inner: InFlightInner::Fdatasync { - sqe: Fdatasync { fd: _ }, + sqe: Fdatasync { fd }, + op_count, }, - blocked, - user_data, - } = self.in_flight.remove(sqe) + .. + } = self.in_flight.get_mut(sqe).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fdatasync") }; - self.complete(Cqe::Fdatasync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Noop { sqe } => { - let InFlight { - inner: InFlightInner::Noop, - blocked, - user_data, - } = self.in_flight.remove(sqe) - else { - unreachable!("invalid sqe: expected noop") - }; + fd.fdatasync([effect]); + *op_count -= 1; + + *op_count == 0 + }; - self.complete(Cqe::Noop { + if is_complete { + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + self.complete(Cqe::Fdatasync { result: Ok(()), user_data, }); self.schedule_linked(sqe, true, blocked); } } + Operation::Noop { sqe } => { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight.remove(sqe) + else { + unreachable!("invalid sqe: expected noop") + }; - return true; + self.complete(Cqe::Noop { + result: Ok(()), + user_data, + }); + self.schedule_linked(sqe, true, blocked); + } } - - false } fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 93e1288fac3..64aadf5a017 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -1,6 +1,6 @@ use alloc::{collections::BTreeMap, sync::Arc}; use core::{ - cmp, fmt, + fmt, sync::atomic::{AtomicU64, Ordering}, }; @@ -41,6 +41,75 @@ impl Page { } } +#[derive(Default)] +struct PageMap { + volatile: BTreeMap>, + durable: BTreeMap>, +} + +impl PageMap { + /// Reset the volatile to the durable state. + fn crash(&mut self) { + self.volatile = self.durable.clone(); + } + + /// Move the page at `index` from the volatile to the durable state. + fn sync(&mut self, index: PageIndex) { + self.durable.insert(index, self.volatile.get(&index).cloned().unwrap()); + } + + /// Get the page at `index` for reading. Uses the durable state. + fn get_page(&self, index: PageIndex) -> Option> { + self.durable.get(&index).cloned() + } + + /// Get the page at `index` for writing, or allocate a new page. + /// Uses the volatile state. + fn get_or_allocate_page(&mut self, index: PageIndex) -> Arc { + Arc::clone(self.volatile.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) + } + + /// Change the allocated space, allocating or deallocating pages as needed. + /// Changes the volatile state only. + fn set_len_volatile(&mut self, new_len: u64) { + Self::set_len(&mut self.volatile, new_len); + } + + /// Like [Self::set_len], but operate on the durable state only. + fn set_len_durable(&mut self, new_len: u64) { + Self::set_len(&mut self.durable, new_len); + } + + fn set_len(page_map: &mut BTreeMap>, new_len: u64) { + use core::cmp::Ordering::*; + + let old_len = page_map.len() as u64; + match new_len.cmp(&old_len) { + Equal => {} + Greater => { + let first_new_page = old_len / PAGE_SIZE_U64; + let end_page = new_len / PAGE_SIZE_U64; + + for index in first_new_page..end_page { + page_map + .entry(PageIndex(index)) + .or_insert_with(|| Arc::new(Page::zeroed())); + } + } + Less => { + let first_removed = PageIndex::from_offset(new_len); + let removed = page_map.split_off(&first_removed); + drop(removed); + } + } + } +} + +pub enum Datasync { + Sector(u64), + Length, +} + /// A memory-backed file. /// /// A [File] is backed by a sparse array of [Page]s. Missing pages are read as @@ -50,26 +119,39 @@ impl Page { /// or written. Writing a page is atomic. #[derive(Clone)] pub struct File { - pages: Arc>>>, - len: Arc, + pages: Arc>, + + volatile_len: Arc, + durable_len: Arc, } impl fmt::Debug for File { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("File").field("len", &self.len).finish() + f.debug_struct("File") + .field("volatile_len", &self.volatile_len) + .field("durable_len", &self.durable_len) + .finish() } } impl File { pub(super) fn new() -> Self { Self { - pages: Arc::new(spin::Mutex::new(BTreeMap::new())), - len: Arc::new(AtomicU64::new(0)), + pages: <_>::default(), + volatile_len: <_>::default(), + durable_len: <_>::default(), } } + /// Simulate a crash by resetting to the durable state. + pub(super) fn crash(&self) { + self.volatile_len + .store(self.durable_len.load(Ordering::Relaxed), Ordering::Relaxed); + self.pages.lock().crash(); + } + pub(super) fn len(&self) -> u64 { - self.len.load(Ordering::Relaxed) + self.volatile_len.load(Ordering::Relaxed) } #[allow(unused)] @@ -84,33 +166,11 @@ impl File { /// Extending allocates pages eagerly as needed. Shrinking drops all pages /// at or beyond the new EOF. pub(super) fn set_len(&self, new_len: u64) -> Result<()> { - use cmp::Ordering::*; - if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } - let old_len = self.len(); - - match new_len.cmp(&old_len) { - Equal => {} - Greater => { - let first_new_page = old_len / PAGE_SIZE_U64; - let end_page = new_len / PAGE_SIZE_U64; - - for index in first_new_page..end_page { - self.get_or_allocate_page(PageIndex(index)); - } - - self.len.store(new_len, Ordering::Relaxed); - } - Less => { - self.len.store(new_len, Ordering::Relaxed); - - let first_removed = PageIndex::from_offset(new_len); - let removed = self.pages.lock().split_off(&first_removed); - drop(removed); - } - } + self.pages.lock().set_len_volatile(new_len); + self.volatile_len.store(new_len, Ordering::Relaxed); Ok(()) } @@ -147,17 +207,39 @@ impl File { .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) .ok_or(Error::OffsetOverflow)?; - self.len.fetch_max(end, Ordering::Relaxed); + self.volatile_len.fetch_max(end, Ordering::Relaxed); Ok(()) } + /// Execute an `fdatasync(2)` operation as a series of [Datasync] effects. + /// + /// The result may or may not leave the durable state in the same state as + /// the volatile state at the time the operation started. + /// + /// It is the caller's responsibility to decide whether the operation is + /// considered successful - a partial operation may report success, or a + /// complete operation may report failure. + pub(super) fn fdatasync(&self, ops: impl IntoIterator) { + for op in ops { + match op { + Datasync::Sector(offset) => { + self.pages.lock().sync(PageIndex(offset)); + } + Datasync::Length => { + let new_durable_len = self.volatile_len.load(Ordering::Relaxed); + self.durable_len.store(new_durable_len, Ordering::Relaxed); + self.pages.lock().set_len_durable(new_durable_len); + } + } + } + } + fn get_page(&self, index: PageIndex) -> Option> { - self.pages.lock().get(&index).cloned() + self.pages.lock().get_page(index) } fn get_or_allocate_page(&self, index: PageIndex) -> Arc { - let mut pages = self.pages.lock(); - Arc::clone(pages.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) + self.pages.lock().get_or_allocate_page(index) } } From 77b3a0bdd87f19e701b9c660d82ff850d805176a Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 3 Sep 2026 11:24:25 +0200 Subject: [PATCH 17/47] Prepare `SqeId` for export --- crates/runtime-core/src/sim/io/executor.rs | 39 +++++++++++++--------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index d2e34404dc0..e9befc20afb 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -414,7 +414,14 @@ impl Cqe { } } -type SqeId = usize; +#[derive(Clone, Copy)] +pub struct SqeId(usize); + +impl SqeId { + fn key(&self) -> usize { + self.0 + } +} // TODO: There is no difference between fsync and fdatasync as long as we don't // have an API to fsync the directory of a file after it was created. @@ -704,7 +711,7 @@ impl Executor { } } let slot = self.in_flight.vacant_entry(); - let (in_flight, ops) = sqe.inner.schedule(slot.key()); + let (in_flight, ops) = sqe.inner.schedule(SqeId(slot.key())); self.executing.extend(ops); slot.insert(InFlight { inner: in_flight, @@ -746,7 +753,7 @@ impl Executor { results, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected write") }; @@ -770,7 +777,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected write") }; @@ -800,7 +807,7 @@ impl Executor { results, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected read") }; @@ -824,7 +831,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected read") }; @@ -845,7 +852,7 @@ impl Executor { inner: InFlightInner::Open { sqe: Open { path } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected open") }; @@ -860,7 +867,7 @@ impl Executor { inner: InFlightInner::Create { sqe: Create { path } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected create") }; @@ -880,7 +887,7 @@ impl Executor { inner: InFlightInner::Stat { sqe: Stat { fd } }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected stat") }; @@ -899,7 +906,7 @@ impl Executor { }, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected fallocate") }; @@ -919,7 +926,7 @@ impl Executor { op_count, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fsync") }; @@ -933,7 +940,7 @@ impl Executor { }; if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); self.complete(Cqe::Fsync { result: Ok(()), user_data, @@ -950,7 +957,7 @@ impl Executor { op_count, }, .. - } = self.in_flight.get_mut(sqe).expect("invalid sqe id") + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { unreachable!("invalid sqe: expected fdatasync") }; @@ -962,7 +969,7 @@ impl Executor { }; if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe); + let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); self.complete(Cqe::Fdatasync { result: Ok(()), user_data, @@ -975,7 +982,7 @@ impl Executor { inner: InFlightInner::Noop, blocked, user_data, - } = self.in_flight.remove(sqe) + } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected noop") }; @@ -1011,7 +1018,7 @@ impl Executor { (LinkKind::Soft, true) | (LinkKind::Hard, _) => { let (inner, ops) = next.schedule(sqe); self.executing.extend(ops); - let slot = self.in_flight.get_mut(sqe).expect("invalid sqe id"); + let slot = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id"); *slot = InFlight { inner, blocked, From c5c796327a4e08b7563b1cf7e6ad7e685436f89c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 3 Sep 2026 17:34:07 +0200 Subject: [PATCH 18/47] WIP: fault injection API --- crates/runtime-core/src/sim/io/executor.rs | 1181 +++++++---------- .../runtime-core/src/sim/io/executor/sqe.rs | 391 ++++++ crates/runtime-core/src/sim/io/fs.rs | 1 + crates/runtime-core/src/sim/io/mod.rs | 12 +- 4 files changed, 903 insertions(+), 682 deletions(-) create mode 100644 crates/runtime-core/src/sim/io/executor/sqe.rs diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index e9befc20afb..9d4237a760b 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -11,351 +11,47 @@ use slab::Slab; use crate::{ io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, sim::{ - io::{ - fs::{self, Datasync}, - Error, Instant, - }, + io::{fs, Error, Instant}, Rng, }, }; -/// Dependency on the previous [Sqe]. -/// -/// A link imposes an ordering constraint: the [Sqe] carrying the link will not -/// be executed before the preceding one completed. Note that a link is only -/// meaningful within a batch of SQEs submitted together. -#[derive(Clone, Copy)] -pub enum LinkKind { - /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. - /// Analogous to `IOSQE_IO_LINK`. - Soft, - /// Run the SQE regardless of the preceding SQE's result. - /// Analoguous to `IOSQE_IO_HARDLINK`. - Hard, -} - -pub struct Sqe { - inner: SqeInner, - link: Option, - user_data: Option, -} - -impl Sqe { - pub fn link(mut self, kind: Option) -> Self { - self.link = kind; - self - } - - pub fn is_linked(&self) -> bool { - self.link.is_some() - } - - pub fn attach(mut self, user_data: T) -> Self { - self.user_data.replace(user_data); - self - } - - pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { - Write { fd, buf, offset }.into() - } - - pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { - Read { fd, buf, offset }.into() - } +pub use crate::sim::io::fs::Datasync; - pub fn open(path: impl AsRef) -> Self { - Open { - path: path.as_ref().into(), - } - .into() - } - - pub fn create(path: impl AsRef) -> Self { - Create { - path: path.as_ref().into(), - } - .into() - } - - pub fn stat(fd: fs::File) -> Self { - Stat { fd }.into() - } - - pub fn fallocate(fd: fs::File, len: u64) -> Self { - Fallocate { fd, total_len: len }.into() - } - - pub fn fsync(fd: fs::File) -> Self { - Fsync { fd }.into() - } - - pub fn fdatasync(fd: fs::File) -> Self { - Fdatasync { fd }.into() - } - - pub fn noop() -> Self { - SqeInner::Noop.into() - } -} +mod sqe; +use sqe::SqeInner; +pub use sqe::{LinkKind, Sqe, SqeId}; -impl> From for Sqe { - fn from(inner: U) -> Self { - Self { - inner: inner.into(), - link: None, - user_data: None, - } - } +// TODO: There is no difference between fsync and fdatasync as long as we don't +// have an API to fsync the directory of a file after it was created. +#[derive(Clone, Copy)] +pub enum FsyncEffect { + Datasync(Datasync), } -enum SqeInner { - Write(Write), - Read(Read), - Open(Open), - Create(Create), - Stat(Stat), - Fallocate(Fallocate), - Fsync(Fsync), - Fdatasync(Fdatasync), +#[derive(Clone, Copy)] +pub enum Operation { + WriteSector(WriteSector), + ReadSector(ReadSector), + Open, + Create, + Stat, + Fallocate, + Fsync { effect: FsyncEffect }, + Fdatasync { effect: Datasync }, Noop, } -impl SqeInner { - fn cancel(self, user_data: Option) -> Cqe { - match self { - SqeInner::Write(..) => Cqe::Write { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Read(..) => Cqe::Read { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Open(..) => Cqe::Open { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Create(..) => Cqe::Create { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Stat(..) => Cqe::Stat { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fallocate(..) => Cqe::Fallocate { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fsync(..) => Cqe::Fsync { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Fdatasync(..) => Cqe::Fdatasync { - result: Err(Error::Cancelled), - user_data, - }, - SqeInner::Noop => Cqe::Noop { - result: Err(Error::Cancelled), - user_data, - }, - } - } - - fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { - match self { - SqeInner::Write(mut sqe) => { - let Write { buf, offset, .. } = &mut sqe; - let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; - - let ops = (0..page_count) - .map(|page| Operation::WriteSector { - sqe: sqe_id, - page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), - }) - .collect::>(); - let op_count = ops.len(); - let write = InFlightInner::Write { - sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (write, ops) - } - SqeInner::Read(mut sqe) => { - let Read { buf, offset, .. } = &mut sqe; - let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; - - let ops = (0..page_count) - .map(|page| Operation::ReadSector { - sqe: sqe_id, - page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), - }) - .collect::>(); - let op_count = ops.len(); - let read = InFlightInner::Read { - sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (read, ops) - } - SqeInner::Open(sqe) => ( - InFlightInner::Open { sqe }, - alloc::vec![Operation::Open { sqe: sqe_id }], - ), - SqeInner::Create(sqe) => ( - InFlightInner::Create { sqe }, - alloc::vec![Operation::Create { sqe: sqe_id }], - ), - SqeInner::Stat(sqe) => ( - InFlightInner::Stat { sqe }, - alloc::vec![Operation::Stat { sqe: sqe_id }], - ), - SqeInner::Fallocate(sqe) => ( - InFlightInner::Fallocate { sqe }, - alloc::vec![Operation::Fallocate { sqe: sqe_id }], - ), - SqeInner::Fsync(sqe) => { - let Fsync { fd } = &sqe; - - let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Sector(offset), - }) - .chain([Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Length, - }]) - .collect::>(); - let in_flight = InFlightInner::Fsync { - sqe, - op_count: ops.len(), - }; - - (in_flight, ops) - } - SqeInner::Fdatasync(sqe) => { - let Fdatasync { fd } = &sqe; - - let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Sector(offset), - }) - .chain([Operation::Fdatasync { - sqe: sqe_id, - effect: Datasync::Length, - }]) - .collect::>(); - let in_flight = InFlightInner::Fdatasync { - sqe, - op_count: ops.len(), - }; - - (in_flight, ops) - } - SqeInner::Noop => (InFlightInner::Noop, alloc::vec![Operation::Noop { sqe: sqe_id }]), - } - } -} - -impl From for SqeInner { - fn from(inner: Write) -> Self { - Self::Write(inner) - } -} - -impl From for SqeInner { - fn from(inner: Read) -> Self { - Self::Read(inner) - } -} - -impl From for SqeInner { - fn from(inner: Open) -> Self { - Self::Open(inner) - } -} - -impl From for SqeInner { - fn from(inner: Create) -> Self { - Self::Create(inner) - } -} - -impl From for SqeInner { - fn from(inner: Stat) -> Self { - Self::Stat(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fallocate) -> Self { - Self::Fallocate(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fsync) -> Self { - Self::Fsync(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fdatasync) -> Self { - Self::Fdatasync(inner) - } -} - -struct Write { - fd: fs::File, - buf: ErasedBoxPtr, - offset: u64, -} - -struct Read { - fd: fs::File, - buf: ErasedBoxPtr, - offset: u64, -} - -struct Open { - path: Box, -} - -struct Create { - path: Box, -} - -struct Stat { - fd: fs::File, -} - -struct Fallocate { - fd: fs::File, - total_len: u64, -} - -struct Fsync { - #[allow(unused)] - fd: fs::File, +#[derive(Clone, Copy)] +pub struct WriteSector { + pub page_offset: usize, + pub buf_offset: usize, } -struct Fdatasync { - #[allow(unused)] - fd: fs::File, +#[derive(Clone, Copy)] +pub struct ReadSector { + pub page_offset: usize, + pub buf_offset: usize, } #[derive(Debug)] @@ -414,122 +110,168 @@ impl Cqe { } } -#[derive(Clone, Copy)] -pub struct SqeId(usize); - -impl SqeId { - fn key(&self) -> usize { - self.0 - } +pub struct Blocked { + pub link: LinkKind, + pub sqe: SqeInner, + pub user_data: Option, } -// TODO: There is no difference between fsync and fdatasync as long as we don't -// have an API to fsync the directory of a file after it was created. -enum FsyncEffect { - Datasync(Datasync), +pub struct InFlight { + pub inner: InFlightInner, + pub blocked: VecDeque>, + pub user_data: Option, } -enum Operation { - WriteSector { - sqe: SqeId, - page_offset: usize, - buf_offset: usize, - }, - ReadSector { - sqe: SqeId, - page_offset: usize, - buf_offset: usize, - }, - Open { - sqe: SqeId, - }, - Create { - sqe: SqeId, - }, - Stat { - sqe: SqeId, - }, - Fallocate { - sqe: SqeId, - }, - Fsync { - sqe: SqeId, - effect: FsyncEffect, - }, - Fdatasync { - sqe: SqeId, - effect: Datasync, - }, - Noop { - sqe: SqeId, - }, -} - -struct Blocked { - link: LinkKind, - sqe: SqeInner, - user_data: Option, -} - -struct InFlight { - inner: InFlightInner, - blocked: VecDeque>, - user_data: Option, -} - -enum InFlightInner { +pub enum InFlightInner { Write { - sqe: Write, + sqe: sqe::Write, op_count: usize, - results: Vec>, + results: Vec>, }, Read { - sqe: Read, + sqe: sqe::Read, op_count: usize, - results: Vec>, + results: Vec>, }, Open { - sqe: Open, + sqe: sqe::Open, }, Create { - sqe: Create, + sqe: sqe::Create, }, Stat { - sqe: Stat, + sqe: sqe::Stat, }, Fallocate { - sqe: Fallocate, + sqe: sqe::Fallocate, }, Fsync { - sqe: Fsync, + sqe: sqe::Fsync, op_count: usize, + results: Vec>, }, Fdatasync { - sqe: Fdatasync, + sqe: sqe::Fdatasync, op_count: usize, + results: Vec>, }, Noop, } -pub enum WriteFault { - /// Misdirect the write to an arbitrary page offset in the file. - Misdirected { page_offset: usize }, - /// Report the write as successful, but don't write anything. - Lost, - /// Report the write as successful, but write less bytes than requested. - Short { write_bytes: usize }, - /// Delay the write until at least `deadline`. - Delayed { deadline: Instant }, - /// Execute the side effects, but never report completion. - NoCompletion, - /// Report an error without executing side effects. - Error(Error), +struct Executing { + sqe: SqeId, + inner: Operation, +} + +impl Executing { + fn traverse(self, f: impl FnOnce(SqeId, Operation) -> Option) -> Option { + let Self { sqe, inner } = self; + f(sqe, inner).map(|inner| Self { sqe, inner }) + } +} + +pub enum Fault { + /// Drop the operation entirely. + Skip, + /// Put the operation back onto the queue for later execution. + Delay(T), + /// Execute a visible effect. + Visible(Effect), +} + +impl Fault { + fn exec_visible(self, f: impl FnOnce(EitherOrBoth)) -> Option { + match self { + Fault::Skip => None, + Fault::Delay(effect) => Some(effect), + Fault::Visible(visible) => { + visible.exec(f); + None + } + } + } +} + +pub enum Effect { + /// Run the operation as normal. + Run(T), + /// Run the effect, but report an injected error. + RunThenError { effect: T, error: Error }, + /// Skip the effect, but report an injected error. + SkipThenError { error: Error }, } -pub trait FaultInjector { - fn maybe_write_fault(&self, rng: &Rng, now: Instant, page_offset: usize) -> Option; +impl Effect { + fn exec(self, f: impl FnOnce(EitherOrBoth)) { + use EitherOrBoth::*; + match self { + Effect::Run(effect) => f(Left(effect)), + Effect::RunThenError { effect, error } => f(Both(effect, error)), + Effect::SkipThenError { error } => f(Right(error)), + } + } +} + +enum EitherOrBoth { + Left(T), + Right(U), + Both(T, U), +} + +impl EitherOrBoth { + fn traverse(self, f: impl FnOnce(T) -> V, g: impl FnOnce(U) -> V) -> V { + match self { + Self::Left(t) => f(t), + Self::Right(u) => g(u), + Self::Both(t, u) => { + f(t); + g(u) + } + } + } +} + +pub trait FaultInjector { + fn inject_write_sector_fault(&mut self, sqe: &InFlight, op: WriteSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_read_sector_fault(&mut self, sqe: &InFlight, op: ReadSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_open_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_create_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_stat_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fallocate_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fsync_fault(&mut self, sqe: &InFlight, op: FsyncEffect) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_fdatasync_fault(&mut self, sqe: &InFlight, op: Datasync) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_noop_fault(&mut self, sqe: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } } +pub struct NoFaults; +impl FaultInjector for NoFaults {} + /// Completion queue overflow policy. /// /// Note that we do **not** model `IORING_FEAT_NODROP`, because we never want @@ -580,7 +322,7 @@ pub struct Executor { completions: VecDeque>, in_flight: Slab>, - executing: VecDeque, + executing: VecDeque, fstree: BTreeMap, fs::File>, @@ -634,20 +376,24 @@ impl Executor { /// to completion. Submissions that were not yet scheduled are dropped. The /// file state remains unchanged. /// + /// Execution is subject to `faults`. If a fault evaluates to [Fault::Skip], + /// that operation is dropped. + /// /// After this method returns, the completion queue is empty. - pub fn restart(&mut self, now: Instant) { + pub fn restart(&mut self, faults: &mut impl FaultInjector) { self.submissions.clear(); let cq_overflow_orig = self.cq_overflow; self.cq_overflow = OnCqOverflow::Drop; let executing = mem::take(&mut self.executing); for op in executing { - self.execute(op, now); + self.execute(op, faults); } self.completions.clear(); self.cq_overflow = cq_overflow_orig; self.cq_dropped = 0; } + /// Submit a batch of [Sqe]s for later execution. pub fn submit(&mut self, sqes: Batch) -> Result<(), Batch::IntoIter> where Batch: IntoIterator>, @@ -675,17 +421,26 @@ impl Executor { self.completions.push_back(cqe); } + /// Number of completions that were dropped due to completion queue overflow + /// over the lifetime of this executor. + /// + /// Always zero if the executor was configured with [OnCqOverflow::Panic]. pub fn dropped_completions(&self) -> usize { self.cq_dropped } + /// Drain the completion queue. pub fn completed(&mut self) -> impl Iterator> { self.completions.drain(..) } - pub fn tick(&mut self, rng: &Rng, now: Instant) -> bool { + /// Drain the submission queue and advance one scheduled operation. + /// + /// The operation to advance is chosen randomly using `rng`. + /// The operation is subject to `faults`. + pub fn tick(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { let mut progress = self.schedule(); - progress |= self.execute_random(rng, now); + progress |= self.execute_random(rng, faults); progress } @@ -725,277 +480,353 @@ impl Executor { progress } - fn execute_random(&mut self, rng: &Rng, now: Instant) -> bool { + fn execute_random(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { if self.executing.is_empty() { return false; } if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { - self.execute(op, now); + if let Some(delay) = self.execute(op, faults) { + self.executing.push_back(delay); + } true } else { false } } - fn execute(&mut self, op: Operation, _now: Instant) { - match op { - Operation::WriteSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected write") - }; - let bytes = buf.as_bytes(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &buf.as_bytes()[buf_offset..end]; - let result = fd.write_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Write { - sqe: Write { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected write") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Write { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::ReadSector { - sqe, - page_offset, - buf_offset, - } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { fd, buf, .. }, - op_count, - results, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected read") - }; - let bytes = buf.as_bytes_mut(); - let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - - let buf = &mut buf.as_bytes_mut()[buf_offset..end]; - let result = fd.read_page(buf, page_offset as _); - results.push(result); - - results.len() == *op_count - }; - - if is_complete { - let InFlight { - inner: - InFlightInner::Read { - sqe: Read { mut buf, .. }, - op_count, - results, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected read") - }; - assert!(results.len() == op_count); - // TODO: Propagate all errors? - // TODO: Allow write op failures and reflect in returned number. - let result = match results.into_iter().find_map(|r| r.map_err(Error::from).err()) { - Some(error) => Err(error), - None => Ok(buf.as_bytes().len()), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Read { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - } - Operation::Open { sqe } => { - let InFlight { - inner: InFlightInner::Open { sqe: Open { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected open") - }; - - let result = self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }); - let is_success = result.is_ok(); - self.complete(Cqe::Open { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Create { sqe } => { - let InFlight { - inner: InFlightInner::Create { sqe: Create { path } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected create") - }; - - let result = match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), - btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { - path: entry.key().clone(), - }), - }; - let is_success = result.is_ok(); - self.complete(Cqe::Create { result, user_data }); - self.schedule_linked(sqe, is_success, blocked); - } - Operation::Stat { sqe } => { - let InFlight { - inner: InFlightInner::Stat { sqe: Stat { fd } }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected stat") - }; - - self.complete(Cqe::Stat { - result: Ok(Statx { size: fd.len() }), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - Operation::Fallocate { sqe } => { - let InFlight { - inner: - InFlightInner::Fallocate { - sqe: Fallocate { fd, total_len }, - }, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected fallocate") - }; - - self.complete(Cqe::Fallocate { - result: fd.set_len(total_len).map_err(Error::from), - user_data, - }); - self.schedule_linked(sqe, true, blocked); + fn execute(&mut self, op: Executing, faults: &mut impl FaultInjector) -> Option { + op.traverse(|sqe, op| { + let in_flight = self.in_flight.get(sqe.key()).expect("invalid sqe id"); + match op { + Operation::WriteSector(effect) => faults + .inject_write_sector_fault(in_flight, effect) + .exec_visible(|eff| self.execute_write_sector(sqe, eff)) + .map(Operation::WriteSector), + Operation::ReadSector(effect) => faults + .inject_read_sector_fault(in_flight, effect) + .exec_visible(|eff| self.execute_read_sector(sqe, eff)) + .map(Operation::ReadSector), + Operation::Open => faults + .inject_open_fault(in_flight) + .exec_visible(|eff| self.execute_open(sqe, eff)) + .map(|()| Operation::Open), + Operation::Create => faults + .inject_create_fault(in_flight) + .exec_visible(|eff| self.execute_create(sqe, eff)) + .map(|()| Operation::Create), + Operation::Stat => faults + .inject_stat_fault(in_flight) + .exec_visible(|eff| self.execute_stat(sqe, eff)) + .map(|()| Operation::Stat), + Operation::Fallocate => faults + .inject_fallocate_fault(in_flight) + .exec_visible(|eff| self.execute_fallocate(sqe, eff)) + .map(|()| Operation::Fallocate), + Operation::Fsync { effect } => faults + .inject_fsync_fault(in_flight, effect) + .exec_visible(|eff| self.execute_fsync(sqe, eff)) + .map(|effect| Operation::Fsync { effect }), + Operation::Fdatasync { effect } => faults + .inject_fdatasync_fault(in_flight, effect) + .exec_visible(|eff| self.execute_fdatasync(sqe, eff)) + .map(|effect| Operation::Fdatasync { effect }), + Operation::Noop => faults + .inject_noop_fault(in_flight) + .exec_visible(|eff| self.execute_noop(sqe, eff)) + .map(|()| Operation::Noop), } - Operation::Fsync { sqe, effect } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Fsync { - sqe: Fsync { fd }, - op_count, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected fsync") - }; - - match effect { - FsyncEffect::Datasync(effect) => fd.fdatasync([effect]), - } - *op_count -= 1; - - *op_count == 0 - }; + }) + } + + fn execute_write_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Write { + sqe: sqe::Write { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected write") + }; + let mut run = |WriteSector { + page_offset, + buf_offset, + }| { + let bytes = buf.as_bytes(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &buf.as_bytes()[buf_offset..end]; + fd.write_page(buf, page_offset as _).map_err(Into::into) + }; + results.push(eff.traverse(run, Err)); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Write { + sqe: sqe::Write { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected write") + }; + assert!(results.len() == op_count); + let bytes_written = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + // TODO: Propagate all errors? + let result = match results.into_iter().find_map(Result::err) { + Some(error) => Err(error), + None => Ok(bytes_written), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Write { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } - if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); - self.complete(Cqe::Fsync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - } - Operation::Fdatasync { sqe, effect } => { - let is_complete = { - let InFlight { - inner: - InFlightInner::Fdatasync { - sqe: Fdatasync { fd }, - op_count, - }, - .. - } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") - else { - unreachable!("invalid sqe: expected fdatasync") - }; + fn execute_read_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Read { + sqe: sqe::Read { fd, buf, .. }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected read") + }; + let mut run = |ReadSector { + page_offset, + buf_offset, + }| { + let bytes = buf.as_bytes_mut(); + let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); + + let buf = &mut buf.as_bytes_mut()[buf_offset..end]; + fd.read_page(buf, page_offset as _).map_err(Into::into) + }; + results.push(eff.traverse(run, Err)); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: + InFlightInner::Read { + sqe: sqe::Read { mut buf, .. }, + op_count, + results, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected read") + }; + assert!(results.len() == op_count); + let bytes_read = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + // TODO: Propagate all errors? + let result = match results.into_iter().find_map(Result::err) { + Some(error) => Err(error), + None => Ok(bytes_read), + }; + let is_success = result.is_ok(); + self.complete(Cqe::Read { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } + fn execute_open(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Open { + sqe: sqe::Open { path }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected open") + }; + let result = eff.traverse( + |()| self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }), + Err, + ); + + let is_success = result.is_ok(); + self.complete(Cqe::Open { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_create(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Create { + sqe: sqe::Create { path }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected create") + }; + let run = |()| match self.fstree.entry(path) { + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { + path: entry.key().clone(), + }), + }; + let result = eff.traverse(run, Err); + let is_success = result.is_ok(); + self.complete(Cqe::Create { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_stat(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Stat { sqe: sqe::Stat { fd } }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected stat") + }; + let result = eff.traverse(|()| Ok(Statx { size: fd.len() }), Err); + let is_success = result.is_ok(); + self.complete(Cqe::Stat { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_fallocate(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Fallocate { + sqe: sqe::Fallocate { fd, total_len }, + }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fallocate") + }; + let result = eff.traverse(|()| fd.set_len(total_len).map_err(Into::into), Err); + let is_success = result.is_ok(); + self.complete(Cqe::Fallocate { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + + fn execute_fsync(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Fsync { + sqe: sqe::Fsync { fd }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fsync") + }; + let result = eff.traverse( + |FsyncEffect::Datasync(effect)| { fd.fdatasync([effect]); - *op_count -= 1; - - *op_count == 0 - }; + Ok(()) + }, + Err, + ); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: InFlightInner::Fsync { results, .. }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fsync") + }; + // TODO: Propagate all errors? + let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let is_success = result.is_ok(); + self.complete(Cqe::Fsync { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + } - if is_complete { - let InFlight { blocked, user_data, .. } = self.in_flight.remove(sqe.key()); - self.complete(Cqe::Fdatasync { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } - } - Operation::Noop { sqe } => { - let InFlight { - inner: InFlightInner::Noop, - blocked, - user_data, - } = self.in_flight.remove(sqe.key()) - else { - unreachable!("invalid sqe: expected noop") - }; - - self.complete(Cqe::Noop { - result: Ok(()), - user_data, - }); - self.schedule_linked(sqe, true, blocked); - } + fn execute_fdatasync(&mut self, sqe: SqeId, eff: EitherOrBoth) { + let is_complete = { + let InFlight { + inner: + InFlightInner::Fdatasync { + sqe: sqe::Fdatasync { fd }, + op_count, + results, + }, + .. + } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") + else { + unreachable!("invalid sqe: expected fdatasync") + }; + let result = eff.traverse( + |effect| { + fd.fdatasync([effect]); + Ok(()) + }, + Err, + ); + results.push(result); + + results.len() == *op_count + }; + + if is_complete { + let InFlight { + inner: InFlightInner::Fdatasync { results, .. }, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected fdatasync") + }; + // TODO: Propagate all errors? + let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let is_success = result.is_ok(); + self.complete(Cqe::Fdatasync { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); } } + fn execute_noop(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { + let InFlight { + inner: InFlightInner::Noop, + blocked, + user_data, + } = self.in_flight.remove(sqe.key()) + else { + unreachable!("invalid sqe: expected noop") + }; + let result = eff.traverse(Ok, Err); + let is_success = result.is_ok(); + self.complete(Cqe::Noop { result, user_data }); + self.schedule_linked(sqe, is_success, blocked); + } + fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { if let Some(Blocked { link, diff --git a/crates/runtime-core/src/sim/io/executor/sqe.rs b/crates/runtime-core/src/sim/io/executor/sqe.rs new file mode 100644 index 00000000000..5b702b4e227 --- /dev/null +++ b/crates/runtime-core/src/sim/io/executor/sqe.rs @@ -0,0 +1,391 @@ +use alloc::{boxed::Box, vec::Vec}; + +use crate::{ + io::{ErasedBoxPtr, SECTOR_SIZE}, + sim::io::{ + executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, WriteSector}, + fs::{self, Datasync}, + Error, + }, +}; + +/// Opaque identifier of a scheduled [Sqe]. +#[derive(Clone, Copy)] +pub struct SqeId(pub(super) usize); + +impl SqeId { + pub(super) fn key(&self) -> usize { + self.0 + } +} + +/// Dependency on the previous [Sqe]. +/// +/// A link imposes an ordering constraint: the [Sqe] carrying the link will not +/// be executed before the preceding one completed. Note that a link is only +/// meaningful within a batch of SQEs submitted together. +#[derive(Clone, Copy)] +pub enum LinkKind { + /// If the preceding SQE failed, cancel this SQE with [Error::Cancelled]. + /// Analogous to `IOSQE_IO_LINK`. + Soft, + /// Run the SQE regardless of the preceding SQE's result. + /// Analoguous to `IOSQE_IO_HARDLINK`. + Hard, +} + +pub struct Sqe { + pub(super) inner: SqeInner, + pub(super) link: Option, + pub(super) user_data: Option, +} + +impl Sqe { + pub fn link(mut self, kind: Option) -> Self { + self.link = kind; + self + } + + pub fn is_linked(&self) -> bool { + self.link.is_some() + } + + pub fn attach(mut self, user_data: T) -> Self { + self.user_data.replace(user_data); + self + } + + pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Write { fd, buf, offset }.into() + } + + pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + Read { fd, buf, offset }.into() + } + + pub fn open(path: impl AsRef) -> Self { + Open { + path: path.as_ref().into(), + } + .into() + } + + pub fn create(path: impl AsRef) -> Self { + Create { + path: path.as_ref().into(), + } + .into() + } + + pub fn stat(fd: fs::File) -> Self { + Stat { fd }.into() + } + + pub fn fallocate(fd: fs::File, len: u64) -> Self { + Fallocate { fd, total_len: len }.into() + } + + pub fn fsync(fd: fs::File) -> Self { + Fsync { fd }.into() + } + + pub fn fdatasync(fd: fs::File) -> Self { + Fdatasync { fd }.into() + } + + pub fn noop() -> Self { + SqeInner::Noop.into() + } +} + +impl> From for Sqe { + fn from(inner: U) -> Self { + Self { + inner: inner.into(), + link: None, + user_data: None, + } + } +} + +pub enum SqeInner { + Write(Write), + Read(Read), + Open(Open), + Create(Create), + Stat(Stat), + Fallocate(Fallocate), + Fsync(Fsync), + Fdatasync(Fdatasync), + Noop, +} + +impl SqeInner { + pub(super) fn cancel(self, user_data: Option) -> Cqe { + match self { + SqeInner::Write(..) => Cqe::Write { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Read(..) => Cqe::Read { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Open(..) => Cqe::Open { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Create(..) => Cqe::Create { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Stat(..) => Cqe::Stat { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fallocate(..) => Cqe::Fallocate { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fsync(..) => Cqe::Fsync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Fdatasync(..) => Cqe::Fdatasync { + result: Err(Error::Cancelled), + user_data, + }, + SqeInner::Noop => Cqe::Noop { + result: Err(Error::Cancelled), + user_data, + }, + } + } + + pub(super) fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { + match self { + SqeInner::Write(mut sqe) => { + let Write { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Executing { + sqe: sqe_id, + inner: Operation::WriteSector(WriteSector { + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }), + }) + .collect::>(); + let op_count = ops.len(); + let write = InFlightInner::Write { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (write, ops) + } + SqeInner::Read(mut sqe) => { + let Read { buf, offset, .. } = &mut sqe; + let buf_len = buf.as_bytes().len(); + let first_sector = (*offset / SECTOR_SIZE as u64) as usize; + let page_count = buf_len / SECTOR_SIZE; + + let ops = (0..page_count) + .map(|page| Executing { + sqe: sqe_id, + inner: Operation::ReadSector(ReadSector { + page_offset: first_sector + page, + buf_offset: *offset as usize + (page * SECTOR_SIZE), + }), + }) + .collect::>(); + let op_count = ops.len(); + let read = InFlightInner::Read { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (read, ops) + } + SqeInner::Open(sqe) => ( + InFlightInner::Open { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Open + }], + ), + SqeInner::Create(sqe) => ( + InFlightInner::Create { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Create + }], + ), + SqeInner::Stat(sqe) => ( + InFlightInner::Stat { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Stat + }], + ), + SqeInner::Fallocate(sqe) => ( + InFlightInner::Fallocate { sqe }, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Fallocate + }], + ), + SqeInner::Fsync(sqe) => { + let Fsync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Sector(offset)), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Length), + }, + }]) + .collect::>(); + let op_count = ops.len(); + let in_flight = InFlightInner::Fsync { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (in_flight, ops) + } + SqeInner::Fdatasync(sqe) => { + let Fdatasync { fd } = &sqe; + + let sector_count = fd.len() / SECTOR_SIZE as u64; + let ops = (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Sector(offset), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Length, + }, + }]) + .collect::>(); + let op_count = ops.len(); + let in_flight = InFlightInner::Fdatasync { + sqe, + op_count, + results: Vec::with_capacity(op_count), + }; + + (in_flight, ops) + } + SqeInner::Noop => ( + InFlightInner::Noop, + alloc::vec![Executing { + sqe: sqe_id, + inner: Operation::Noop + }], + ), + } + } +} + +impl From for SqeInner { + fn from(inner: Write) -> Self { + Self::Write(inner) + } +} + +impl From for SqeInner { + fn from(inner: Read) -> Self { + Self::Read(inner) + } +} + +impl From for SqeInner { + fn from(inner: Open) -> Self { + Self::Open(inner) + } +} + +impl From for SqeInner { + fn from(inner: Create) -> Self { + Self::Create(inner) + } +} + +impl From for SqeInner { + fn from(inner: Stat) -> Self { + Self::Stat(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fallocate) -> Self { + Self::Fallocate(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fsync) -> Self { + Self::Fsync(inner) + } +} + +impl From for SqeInner { + fn from(inner: Fdatasync) -> Self { + Self::Fdatasync(inner) + } +} + +pub struct Write { + pub fd: fs::File, + pub buf: ErasedBoxPtr, + pub offset: u64, +} + +pub struct Read { + pub fd: fs::File, + pub buf: ErasedBoxPtr, + pub offset: u64, +} + +pub struct Open { + pub path: Box, +} + +pub struct Create { + pub path: Box, +} + +pub struct Stat { + pub fd: fs::File, +} + +pub struct Fallocate { + pub fd: fs::File, + pub total_len: u64, +} + +pub struct Fsync { + pub fd: fs::File, +} + +pub struct Fdatasync { + pub fd: fs::File, +} diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 64aadf5a017..fd793300e34 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -105,6 +105,7 @@ impl PageMap { } } +#[derive(Clone, Copy)] pub enum Datasync { Sector(u64), Length, diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index f887ad6cf3e..4cb9003d3e1 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -10,7 +10,7 @@ use slab::Slab; use crate::{ io::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}, - sim::Rng, + sim::{io::executor::FaultInjector, Rng}, }; mod executor; @@ -60,12 +60,12 @@ pub struct SimulatorIO { } impl SimulatorIO { - pub fn tick(&self, rng: &Rng, now: Instant) -> bool { + pub fn tick(&self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let mut buffers = self.inner.buffers.lock(); - let mut progress = executor.tick(rng, now); + let mut progress = executor.tick(rng, faults); for cqe in executor.completed() { let completion = pending.remove(cqe.user_data().unwrap()); match cqe { @@ -376,7 +376,7 @@ fn reify( #[cfg(test)] mod tests { - use crate::sim::{time::TimeHandle, GlobalRng}; + use crate::sim::{io::executor::NoFaults, GlobalRng}; use super::*; @@ -384,7 +384,6 @@ mod tests { rt: tokio::runtime::LocalRuntime, io: SimulatorIO, rng: Rng, - time: TimeHandle, } impl Runtime { @@ -395,13 +394,12 @@ mod tests { .unwrap(), io: SimulatorIO::default(), rng: GlobalRng::new(0), - time: TimeHandle::default(), } } fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); - while self.io.tick(&self.rng, self.time.now()) {} + while self.io.tick(&self.rng, &mut NoFaults) {} self.rt.block_on(fut).unwrap() } } From f25388d94f608cf1afd97d214b060ab7ce690363 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 4 Sep 2026 16:55:39 +0200 Subject: [PATCH 19/47] Consider actual size written/read for Write/Read completions --- crates/runtime-core/src/io/buf.rs | 8 ++++++++ crates/runtime-core/src/sim/io/mod.rs | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-core/src/io/buf.rs index 16a8123b5c5..14b88f05c9e 100644 --- a/crates/runtime-core/src/io/buf.rs +++ b/crates/runtime-core/src/io/buf.rs @@ -115,6 +115,14 @@ mod boxed { len: self.len, } } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } } impl Drop for ErasedBox { diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index 4cb9003d3e1..ce01d559fcb 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -75,7 +75,14 @@ impl SimulatorIO { }; let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(_written) => Ok(erased_buf), + Ok(written) if written == erased_buf.len() => Ok(erased_buf), + Ok(written) => Err(ErrorWith { + error: Error::ShortWrite { + expected: erased_buf.len(), + written, + }, + with: erased_buf, + }), Err(error) => Err(ErrorWith { error, with: erased_buf, @@ -89,7 +96,14 @@ impl SimulatorIO { }; let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(_written) => Ok(erased_buf), + Ok(read) if read == erased_buf.len() => Ok(erased_buf), + Ok(read) => Err(ErrorWith { + error: Error::UnexpectedEof { + expected: erased_buf.len(), + read, + }, + with: erased_buf, + }), Err(error) => Err(ErrorWith { error, with: erased_buf, From 6c230904cb6616f05c0c2db3a902bd9570e9b50c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 07:37:49 +0200 Subject: [PATCH 20/47] Turns out we can do without the unsafe, ownership-erased `ErasedPtr` --- crates/runtime-core/src/io/buf.rs | 33 ++++----- crates/runtime-core/src/io/mod.rs | 2 +- crates/runtime-core/src/sim/io/executor.rs | 17 +++-- .../runtime-core/src/sim/io/executor/sqe.rs | 32 ++++++--- crates/runtime-core/src/sim/io/mod.rs | 69 ++++++++----------- 5 files changed, 75 insertions(+), 78 deletions(-) diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-core/src/io/buf.rs index 14b88f05c9e..35388c62a25 100644 --- a/crates/runtime-core/src/io/buf.rs +++ b/crates/runtime-core/src/io/buf.rs @@ -66,6 +66,7 @@ mod boxed { use crate::io::AlignedBytes; /// A type-erased [AlignedBytes] heap allocation. + #[derive(Debug)] pub struct ErasedBox { ptr: NonNull, len: usize, @@ -109,11 +110,8 @@ mod boxed { boxed } - pub fn as_ptr(&self) -> ErasedBoxPtr { - ErasedBoxPtr { - ptr: self.ptr.as_ptr(), - len: self.len, - } + pub fn as_mut_ptr(&self) -> *mut u8 { + self.ptr.as_ptr() } pub fn len(&self) -> usize { @@ -123,26 +121,19 @@ mod boxed { pub fn is_empty(&self) -> bool { self.len == 0 } - } - impl Drop for ErasedBox { - fn drop(&mut self) { - unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } } - } - pub struct ErasedBoxPtr { - ptr: *mut u8, - len: usize, - } - - impl ErasedBoxPtr { - pub fn as_bytes(&mut self) -> &[u8] { - unsafe { core::slice::from_raw_parts(self.ptr, self.len) } + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } } + } - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { core::slice::from_raw_parts_mut(self.ptr, self.len) } + impl Drop for ErasedBox { + fn drop(&mut self) { + unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } } } @@ -183,4 +174,4 @@ mod boxed { } } #[cfg(feature = "alloc")] -pub use boxed::{ErasedBox, ErasedBoxPtr}; +pub use boxed::ErasedBox; diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-core/src/io/mod.rs index d5a946b14ca..76979ba9e79 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-core/src/io/mod.rs @@ -1,7 +1,7 @@ mod buf; pub use buf::AlignedBytes; #[cfg(feature = "alloc")] -pub use buf::{ErasedBox, ErasedBoxPtr}; +pub use buf::ErasedBox; mod error; pub use error::ErrorWith; diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index 9d4237a760b..12086398379 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -9,7 +9,7 @@ use core::{mem, num::NonZeroUsize, result::Result}; use slab::Slab; use crate::{ - io::{ErasedBoxPtr, Statx, SECTOR_SIZE}, + io::{ErasedBox, Statx, SECTOR_SIZE}, sim::{ io::{fs, Error, Instant}, Rng, @@ -58,10 +58,12 @@ pub struct ReadSector { pub enum Cqe { Write { result: Result, + buf: ErasedBox, user_data: Option, }, Read { result: Result, + buf: ErasedBox, user_data: Option, }, Open { @@ -484,9 +486,10 @@ impl Executor { if self.executing.is_empty() { return false; } - if let Some(op) = self.executing.remove(rng.index(self.executing.len())) { + let index = rng.index(self.executing.len()); + if let Some(op) = self.executing.remove(index) { if let Some(delay) = self.execute(op, faults) { - self.executing.push_back(delay); + self.executing.insert(index, delay); } true } else { @@ -571,7 +574,7 @@ impl Executor { let InFlight { inner: InFlightInner::Write { - sqe: sqe::Write { mut buf, .. }, + sqe: sqe::Write { buf, .. }, op_count, results, }, @@ -589,7 +592,7 @@ impl Executor { None => Ok(bytes_written), }; let is_success = result.is_ok(); - self.complete(Cqe::Write { result, user_data }); + self.complete(Cqe::Write { result, buf, user_data }); self.schedule_linked(sqe, is_success, blocked); } } @@ -627,7 +630,7 @@ impl Executor { let InFlight { inner: InFlightInner::Read { - sqe: sqe::Read { mut buf, .. }, + sqe: sqe::Read { buf, .. }, op_count, results, }, @@ -645,7 +648,7 @@ impl Executor { None => Ok(bytes_read), }; let is_success = result.is_ok(); - self.complete(Cqe::Read { result, user_data }); + self.complete(Cqe::Read { result, buf, user_data }); self.schedule_linked(sqe, is_success, blocked); } } diff --git a/crates/runtime-core/src/sim/io/executor/sqe.rs b/crates/runtime-core/src/sim/io/executor/sqe.rs index 5b702b4e227..e29bf0ceeae 100644 --- a/crates/runtime-core/src/sim/io/executor/sqe.rs +++ b/crates/runtime-core/src/sim/io/executor/sqe.rs @@ -1,7 +1,7 @@ use alloc::{boxed::Box, vec::Vec}; use crate::{ - io::{ErasedBoxPtr, SECTOR_SIZE}, + io::{ErasedBox, SECTOR_SIZE}, sim::io::{ executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, WriteSector}, fs::{self, Datasync}, @@ -35,7 +35,7 @@ pub enum LinkKind { } pub struct Sqe { - pub(super) inner: SqeInner, + pub(crate) inner: SqeInner, pub(super) link: Option, pub(super) user_data: Option, } @@ -55,11 +55,11 @@ impl Sqe { self } - pub fn write(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + pub fn write(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { Write { fd, buf, offset }.into() } - pub fn read(fd: fs::File, buf: ErasedBoxPtr, offset: u64) -> Self { + pub fn read(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { Read { fd, buf, offset }.into() } @@ -96,6 +96,20 @@ impl Sqe { pub fn noop() -> Self { SqeInner::Noop.into() } + + /// Extract the [ErasedBox] buffer if the [Sqe] carries one. + pub(crate) fn into_buf(self) -> Option { + match self.inner { + SqeInner::Write(Write { buf, .. }) | SqeInner::Read(Read { buf, .. }) => Some(buf), + SqeInner::Open { .. } + | SqeInner::Create { .. } + | SqeInner::Stat { .. } + | SqeInner::Fallocate { .. } + | SqeInner::Fsync { .. } + | SqeInner::Fdatasync { .. } + | SqeInner::Noop => None, + } + } } impl> From for Sqe { @@ -123,12 +137,14 @@ pub enum SqeInner { impl SqeInner { pub(super) fn cancel(self, user_data: Option) -> Cqe { match self { - SqeInner::Write(..) => Cqe::Write { + SqeInner::Write(Write { buf, .. }) => Cqe::Write { result: Err(Error::Cancelled), + buf, user_data, }, - SqeInner::Read(..) => Cqe::Read { + SqeInner::Read(Read { buf, .. }) => Cqe::Read { result: Err(Error::Cancelled), + buf, user_data, }, SqeInner::Open(..) => Cqe::Open { @@ -355,14 +371,14 @@ impl From for SqeInner { pub struct Write { pub fd: fs::File, - pub buf: ErasedBoxPtr, + pub buf: ErasedBox, pub offset: u64, } pub struct Read { pub fd: fs::File, - pub buf: ErasedBoxPtr, pub offset: u64, + pub buf: ErasedBox, } pub struct Open { diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index ce01d559fcb..de9e66da5b1 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -63,51 +63,42 @@ impl SimulatorIO { pub fn tick(&self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); - let mut buffers = self.inner.buffers.lock(); let mut progress = executor.tick(rng, faults); for cqe in executor.completed() { let completion = pending.remove(cqe.user_data().unwrap()); match cqe { - Cqe::Write { result, .. } => { - let CompletionHandle::Write { tx, buf_key } = completion else { + Cqe::Write { result, buf, .. } => { + let CompletionHandle::Write { tx } = completion else { unreachable!("invalid cqe / completion pairing") }; - let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(written) if written == erased_buf.len() => Ok(erased_buf), + Ok(written) if written == buf.len() => Ok(buf), Ok(written) => Err(ErrorWith { error: Error::ShortWrite { - expected: erased_buf.len(), + expected: buf.len(), written, }, - with: erased_buf, - }), - Err(error) => Err(ErrorWith { - error, - with: erased_buf, + with: buf, }), + Err(error) => Err(ErrorWith { error, with: buf }), }; let _ = tx.send(result); } - Cqe::Read { result, .. } => { - let CompletionHandle::Read { tx, buf_key } = completion else { + Cqe::Read { result, buf, .. } => { + let CompletionHandle::Read { tx } = completion else { unreachable!("invalid cqe / completion pairing") }; - let erased_buf = buffers.remove(buf_key); let result = match result { - Ok(read) if read == erased_buf.len() => Ok(erased_buf), + Ok(read) if read == buf.len() => Ok(buf), Ok(read) => Err(ErrorWith { error: Error::UnexpectedEof { - expected: erased_buf.len(), + expected: buf.len(), read, }, - with: erased_buf, - }), - Err(error) => Err(ErrorWith { - error, - with: erased_buf, + with: buf, }), + Err(error) => Err(ErrorWith { error, with: buf }), }; let _ = tx.send(result); } @@ -187,8 +178,7 @@ impl SimulatorIO { fn submit_with( &self, sqe: Sqe, - buf: ErasedBox, - completion_handle: impl FnOnce(CompletionSender>, usize) -> CompletionHandle, + completion_handle: impl FnOnce(CompletionSender>) -> CompletionHandle, ) -> Completion>> { let (tx, rx) = oneshot::channel(); @@ -197,15 +187,20 @@ impl SimulatorIO { let pending_entry = pending.vacant_entry(); match executor.submit([sqe.attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(ErrorWith { + Err(mut sqe) => { + let buf = sqe + .next() + .expect("submitted one sqe therefore one must be returned on overflow") + .into_buf() + .expect("sqe must have been buffer-carrying"); + tx.send(Err(ErrorWith { error: Error::SubmissionQueueOverflow, with: buf, })) - .unwrap_or_else(|_| unreachable!("rx is alive")), + .unwrap_or_else(|_| unreachable!("rx is alive")) + } Ok(()) => { - let buf_key = self.inner.buffers.lock().insert(buf); - pending_entry.insert(completion_handle(tx, buf_key)); + pending_entry.insert(completion_handle(tx)); } } @@ -216,7 +211,6 @@ impl SimulatorIO { struct SimulatorInner { executor: spin::Mutex>, pending: spin::Mutex>, - buffers: Arc>>, } impl Default for SimulatorInner { @@ -224,7 +218,6 @@ impl Default for SimulatorInner { Self { executor: spin::Mutex::new(Executor::new(<_>::default())), pending: <_>::default(), - buffers: <_>::default(), } } } @@ -289,11 +282,9 @@ type CompletionSender = oneshot::Sender>; enum CompletionHandle { Write { tx: CompletionSender>, - buf_key: usize, }, Read { tx: CompletionSender>, - buf_key: usize, }, Open { tx: CompletionSender, @@ -339,10 +330,8 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let erased_buf = ErasedBox::from_aligned(buf); - let buf_ptr = erased_buf.as_ptr(); - self.submit_with(Sqe::write(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { - CompletionHandle::Write { tx, buf_key } + self.submit_with(Sqe::write(fd, ErasedBox::from_aligned(buf), offset), |tx| { + CompletionHandle::Write { tx } }) } @@ -352,10 +341,8 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - let erased_buf = ErasedBox::from_aligned(buf); - let buf_ptr = erased_buf.as_ptr(); - self.submit_with(Sqe::read(fd, buf_ptr, offset), erased_buf, |tx, buf_key| { - CompletionHandle::Read { tx, buf_key } + self.submit_with(Sqe::read(fd, ErasedBox::from_aligned(buf), offset), |tx| { + CompletionHandle::Read { tx } }) } @@ -462,6 +449,6 @@ mod tests { buf.clear(); let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); - assert!(buf.0.iter().all(|&b| b == 22)); + assert!(buf.0.iter().all(|&b| b == 22),); } } From 08397f6e3ce979dcd171befd1bc3d9258858ec72 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 08:19:17 +0200 Subject: [PATCH 21/47] Fix tests: file reads must actually use the volatile state --- crates/runtime-core/src/sim/io/executor.rs | 4 ++-- crates/runtime-core/src/sim/io/fs.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index 12086398379..c653bba126d 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -360,7 +360,7 @@ impl Executor { /// All submitted and executing operations are cancelled, and files reset to /// their durable state. After this method returns, the completion queue is /// empty. - pub fn crash(&mut self) { + pub fn power_loss(&mut self) { self.submissions.clear(); self.completions.clear(); self.in_flight.clear(); @@ -368,7 +368,7 @@ impl Executor { self.cq_dropped = 0; for file in self.fstree.values_mut() { - file.crash(); + file.power_loss(); } } diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index fd793300e34..a9162470087 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -49,7 +49,7 @@ struct PageMap { impl PageMap { /// Reset the volatile to the durable state. - fn crash(&mut self) { + fn power_loss(&mut self) { self.volatile = self.durable.clone(); } @@ -58,9 +58,9 @@ impl PageMap { self.durable.insert(index, self.volatile.get(&index).cloned().unwrap()); } - /// Get the page at `index` for reading. Uses the durable state. + /// Get the page at `index` for reading. Uses the volatile state. fn get_page(&self, index: PageIndex) -> Option> { - self.durable.get(&index).cloned() + self.volatile.get(&index).cloned() } /// Get the page at `index` for writing, or allocate a new page. @@ -145,10 +145,10 @@ impl File { } /// Simulate a crash by resetting to the durable state. - pub(super) fn crash(&self) { + pub(super) fn power_loss(&self) { self.volatile_len .store(self.durable_len.load(Ordering::Relaxed), Ordering::Relaxed); - self.pages.lock().crash(); + self.pages.lock().power_loss(); } pub(super) fn len(&self) -> u64 { From ade9e51d538b8253e63c287d39500b4c2d416108 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 08:27:34 +0200 Subject: [PATCH 22/47] Fix buf offset --- crates/runtime-core/src/sim/io/executor/sqe.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor/sqe.rs b/crates/runtime-core/src/sim/io/executor/sqe.rs index e29bf0ceeae..ca0ae8f9223 100644 --- a/crates/runtime-core/src/sim/io/executor/sqe.rs +++ b/crates/runtime-core/src/sim/io/executor/sqe.rs @@ -191,7 +191,7 @@ impl SqeInner { sqe: sqe_id, inner: Operation::WriteSector(WriteSector { page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), + buf_offset: page * SECTOR_SIZE, }), }) .collect::>(); @@ -215,7 +215,7 @@ impl SqeInner { sqe: sqe_id, inner: Operation::ReadSector(ReadSector { page_offset: first_sector + page, - buf_offset: *offset as usize + (page * SECTOR_SIZE), + buf_offset: page * SECTOR_SIZE, }), }) .collect::>(); From 6a7679c7439c1d60e1e830f0719afe6e36fd5841 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 08:28:10 +0200 Subject: [PATCH 23/47] Fix file length ops --- crates/runtime-core/src/sim/io/fs.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index a9162470087..88dd69c8c04 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -71,19 +71,18 @@ impl PageMap { /// Change the allocated space, allocating or deallocating pages as needed. /// Changes the volatile state only. - fn set_len_volatile(&mut self, new_len: u64) { - Self::set_len(&mut self.volatile, new_len); + fn set_len_volatile(&mut self, old_len: u64, new_len: u64) { + Self::set_len(&mut self.volatile, old_len, new_len); } - /// Like [Self::set_len], but operate on the durable state only. - fn set_len_durable(&mut self, new_len: u64) { - Self::set_len(&mut self.durable, new_len); + /// Like [Self::set_len_volatile], but operate on the durable state only. + fn set_len_durable(&mut self, old_len: u64, new_len: u64) { + Self::set_len(&mut self.durable, old_len, new_len); } - fn set_len(page_map: &mut BTreeMap>, new_len: u64) { + fn set_len(page_map: &mut BTreeMap>, old_len: u64, new_len: u64) { use core::cmp::Ordering::*; - let old_len = page_map.len() as u64; match new_len.cmp(&old_len) { Equal => {} Greater => { @@ -170,7 +169,9 @@ impl File { if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } - self.pages.lock().set_len_volatile(new_len); + self.pages + .lock() + .set_len_volatile(self.volatile_len.load(Ordering::Relaxed), new_len); self.volatile_len.store(new_len, Ordering::Relaxed); Ok(()) @@ -229,8 +230,8 @@ impl File { } Datasync::Length => { let new_durable_len = self.volatile_len.load(Ordering::Relaxed); - self.durable_len.store(new_durable_len, Ordering::Relaxed); - self.pages.lock().set_len_durable(new_durable_len); + let old_durable_len = self.durable_len.swap(new_durable_len, Ordering::Relaxed); + self.pages.lock().set_len_durable(old_durable_len, new_durable_len); } } } From 9178efd3f6ebd2c9ae0702c3541299004fe41ac1 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 09:47:22 +0200 Subject: [PATCH 24/47] Tests --- crates/runtime-core/src/sim/io/executor.rs | 8 +- crates/runtime-core/src/sim/io/fs.rs | 20 ++-- crates/runtime-core/src/sim/io/mod.rs | 113 +++++++++++++++++++-- 3 files changed, 124 insertions(+), 17 deletions(-) diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-core/src/sim/io/executor.rs index c653bba126d..ead10376542 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-core/src/sim/io/executor.rs @@ -128,12 +128,12 @@ pub enum InFlightInner { Write { sqe: sqe::Write, op_count: usize, - results: Vec>, + results: Vec>, }, Read { sqe: sqe::Read, op_count: usize, - results: Vec>, + results: Vec>, }, Open { sqe: sqe::Open, @@ -585,7 +585,7 @@ impl Executor { unreachable!("invalid sqe: expected write") }; assert!(results.len() == op_count); - let bytes_written = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + let bytes_written = results.iter().filter_map(|r| r.as_ref().ok()).sum(); // TODO: Propagate all errors? let result = match results.into_iter().find_map(Result::err) { Some(error) => Err(error), @@ -641,7 +641,7 @@ impl Executor { unreachable!("invalid sqe: expected read") }; assert!(results.len() == op_count); - let bytes_read = results.iter().filter(|r| r.is_ok()).count() * SECTOR_SIZE; + let bytes_read = results.iter().filter_map(|r| r.as_ref().ok()).sum(); // TODO: Propagate all errors? let result = match results.into_iter().find_map(Result::err) { Some(error) => Err(error), diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-core/src/sim/io/fs.rs index 88dd69c8c04..2b6db7ee2c5 100644 --- a/crates/runtime-core/src/sim/io/fs.rs +++ b/crates/runtime-core/src/sim/io/fs.rs @@ -178,11 +178,17 @@ impl File { } /// Read one complete page. - pub(super) fn read_page(&self, dst: &mut [u8], index: u64) -> Result<()> { + pub(super) fn read_page(&self, dst: &mut [u8], index: u64) -> Result { if dst.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } + let offset = index.checked_mul(PAGE_SIZE as u64).ok_or(Error::OffsetOverflow)?; + let len = self.volatile_len.load(Ordering::Relaxed); + if offset >= len { + return Ok(0); + } + match self.get_page(PageIndex(index)) { Some(page) => { dst.copy_from_slice(&*page.bytes.lock()); @@ -192,26 +198,26 @@ impl File { } } - Ok(()) + Ok(PAGE_SIZE) } /// Write one complete page. - pub(super) fn write_page(&self, src: &[u8], index: u64) -> Result<()> { + pub(super) fn write_page(&self, src: &[u8], index: u64) -> Result { if src.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } - let page = self.get_or_allocate_page(PageIndex(index)); - page.bytes.lock().copy_from_slice(src); - let end = index .checked_add(1) .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) .ok_or(Error::OffsetOverflow)?; + let page = self.get_or_allocate_page(PageIndex(index)); + page.bytes.lock().copy_from_slice(src); + self.volatile_len.fetch_max(end, Ordering::Relaxed); - Ok(()) + Ok(src.len()) } /// Execute an `fdatasync(2)` operation as a series of [Datasync] effects. diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-core/src/sim/io/mod.rs index de9e66da5b1..9970a64e101 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-core/src/sim/io/mod.rs @@ -152,6 +152,10 @@ impl SimulatorIO { progress } + pub fn power_loss(&self) { + self.inner.executor.lock().power_loss(); + } + fn submit( &self, sqe: Sqe, @@ -403,6 +407,10 @@ mod tests { while self.io.tick(&self.rng, &mut NoFaults) {} self.rt.block_on(fut).unwrap() } + + fn power_loss(&self) { + self.io.power_loss(); + } } #[test] @@ -413,15 +421,15 @@ mod tests { #[derive(Debug)] #[repr(C, align(4096))] - struct Buf([u8; 2 * SECTOR_SIZE]); + struct Buf([u8; N]); - impl Buf { + impl Buf { fn clear(&mut self) { self.0.fill(0); } } - impl AlignedBytes for Buf { + impl AlignedBytes for Buf { fn as_bytes(&self) -> &[u8] { &self.0 } @@ -431,8 +439,8 @@ mod tests { } fn from_bytes(b: &[u8]) -> Self { - assert_eq!(b.len(), 2 * SECTOR_SIZE); - let mut buf = [0; 2 * SECTOR_SIZE]; + assert_eq!(b.len(), N); + let mut buf = [0; N]; buf.copy_from_slice(b); Self(buf) } @@ -449,6 +457,99 @@ mod tests { buf.clear(); let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); - assert!(buf.0.iter().all(|&b| b == 22),); + assert_eq!(buf.0, [22; 2 * SECTOR_SIZE]); + } + + #[test] + fn write_read_at_offset() { + let rt = Runtime::new(); + + let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let buf = { + let mut buf = Buf([0; SECTOR_SIZE]); + for i in 0usize..2 { + buf.0.fill((i + 1) as u8 * 2); + let offset = (i * SECTOR_SIZE) as u64; + buf = rt + .run(|io| io.write_all_at(fd.clone(), buf, offset)) + .map_err(ErrorWith::into_err) + .unwrap(); + } + + buf.clear(); + buf + }; + let buf = rt.run(|io| io.read_exact_at(fd, buf, SECTOR_SIZE as u64)).unwrap(); + + assert_eq!(buf.0, [4; SECTOR_SIZE]); + } + + #[test] + fn preallocate() { + let rt = Runtime::new(); + + let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + rt.run(|io| io.reserve(fd.clone(), 2 * SECTOR_SIZE as u64)).unwrap(); + + // Check that reserved space reads as zeroes. + let buf = rt + .run(|io| io.read_exact_at(fd.clone(), Buf([1; 2 * SECTOR_SIZE]), 0)) + .unwrap(); + assert_eq!(buf.0, [0; 2 * SECTOR_SIZE]); + + // The length is reported as the preallocated length. + let stat = rt.run(|io| io.statx(fd.clone())).unwrap(); + assert_eq!(stat.size, 2 * SECTOR_SIZE as u64); + + // Overwriting the second sector works. + let buf = rt + .run(|io| io.write_all_at(fd.clone(), Buf([42; SECTOR_SIZE]), SECTOR_SIZE as u64)) + .unwrap(); + let buf = rt + .run(|io| io.read_exact_at(fd.clone(), buf, SECTOR_SIZE as u64)) + .unwrap(); + assert_eq!(buf.0, [42; SECTOR_SIZE]); + // The first sector still reads as zeroes. + let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); + assert_eq!(buf.0, [0; SECTOR_SIZE]); + } + + #[test] + fn open_succeeds_after_create() { + let rt = Runtime::new(); + + matches!(rt.run(|io| io.open_file("/data/test")), Err(Error::FileNotFound { .. })); + rt.run(|io| io.create_file("/data/test")).unwrap(); + assert!(rt.run(|io| io.open_file("/data/test")).is_ok()); + } + + #[test] + fn unsynced_data_is_lost_after_power_loss() { + let rt = Runtime::new(); + + let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let mut buf = rt + .run(|io| io.write_all_at(fd.clone(), Buf([1; SECTOR_SIZE]), 0)) + .map_err(ErrorWith::into_err) + .unwrap(); + buf.clear(); + + rt.run(|io| io.fdatasync(fd.clone())).unwrap(); + + let mut buf = rt + .run(|io| io.write_all_at(fd.clone(), Buf([2; SECTOR_SIZE]), SECTOR_SIZE as u64)) + .map_err(ErrorWith::into_err) + .unwrap(); + buf.clear(); + + rt.power_loss(); + + let buf = rt.run(|io| io.read_exact_at(fd.clone(), buf, 0)).unwrap(); + assert_eq!(buf.0, [1; SECTOR_SIZE]); + matches!( + rt.run(|io| io.read_exact_at(fd.clone(), buf, SECTOR_SIZE as u64)) + .map_err(ErrorWith::into_err), + Err(Error::UnexpectedEof { .. }) + ); } } From bebc3dc8141aabeaad389bc58586c808c9380aec Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 11:48:46 +0200 Subject: [PATCH 25/47] Move I/O to separate runtime-io crate --- Cargo.lock | 10 +++ Cargo.toml | 2 + crates/runtime-core/Cargo.toml | 9 +- crates/runtime-core/src/lib.rs | 2 - crates/runtime-core/src/sim/mod.rs | 1 - crates/runtime-io/Cargo.toml | 24 ++++++ .../src/io => runtime-io/src}/buf.rs | 8 +- .../src/io => runtime-io/src}/error.rs | 0 .../src/io/mod.rs => runtime-io/src/lib.rs} | 10 ++- .../sim/io => runtime-io/src/sim}/executor.rs | 85 ++++++++++--------- .../io => runtime-io/src/sim}/executor/sqe.rs | 7 +- .../src/sim/io => runtime-io/src/sim}/fs.rs | 0 .../src/sim/io => runtime-io/src/sim}/mod.rs | 47 +++++++--- crates/runtime/Cargo.toml | 1 + crates/runtime/src/io/tokio.rs | 2 +- 15 files changed, 138 insertions(+), 70 deletions(-) create mode 100644 crates/runtime-io/Cargo.toml rename crates/{runtime-core/src/io => runtime-io/src}/buf.rs (97%) rename crates/{runtime-core/src/io => runtime-io/src}/error.rs (100%) rename crates/{runtime-core/src/io/mod.rs => runtime-io/src/lib.rs} (96%) rename crates/{runtime-core/src/sim/io => runtime-io/src/sim}/executor.rs (91%) rename crates/{runtime-core/src/sim/io => runtime-io/src/sim}/executor/sqe.rs (99%) rename crates/{runtime-core/src/sim/io => runtime-io/src/sim}/fs.rs (100%) rename crates/{runtime-core/src/sim/io => runtime-io/src/sim}/mod.rs (91%) diff --git a/Cargo.lock b/Cargo.lock index cdfcdb19b23..a3ace157e6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8532,6 +8532,7 @@ dependencies = [ "futures", "libc", "spacetimedb-runtime-core", + "spacetimedb-runtime-io", "static_assertions", "tokio", "windows-sys 0.61.2", @@ -8542,8 +8543,17 @@ name = "spacetimedb-runtime-core" version = "2.10.0" dependencies = [ "async-task", + "spin", +] + +[[package]] +name = "spacetimedb-runtime-io" +version = "2.10.0" +dependencies = [ "futures-channel", "slab", + "spacetimedb-runtime-core", + "spacetimedb-runtime-io", "spin", "thiserror 2.0.17", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 9f7852863c8..9227df3b2a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ members = [ "crates/query", "crates/runtime", "crates/runtime-core", + "crates/runtime-io", "crates/sats", "crates/schema", "crates/smoketests", @@ -175,6 +176,7 @@ spacetimedb-query = { path = "crates/query", version = "=2.10.0" } spacetimedb-query-builder = { path = "crates/query-builder", version = "=2.10.0" } spacetimedb-runtime = { path = "crates/runtime", version = "=2.10.0" } spacetimedb-runtime-core = { path = "crates/runtime-core", version = "=2.10.0" } +spacetimedb-runtime-io = { path = "crates/runtime-io", version = "=2.10.0" } spacetimedb-sats = { path = "crates/sats", version = "=2.10.0" } spacetimedb-schema = { path = "crates/schema", version = "=2.10.0" } spacetimedb-snapshot = { path = "crates/snapshot", version = "=2.10.0" } diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 5b2b8571106..962883cb127 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -12,15 +12,8 @@ workspace = true [features] default = [] alloc = [] -sim = ["alloc", "dep:async-task", "dep:futures-channel", "dep:slab", "dep:spin"] +sim = ["alloc", "dep:async-task", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } -futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true } -slab = { version = "0.4", default-features = false, optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } -thiserror = { version = "2.0", default-features = false } -zerocopy = "0.8" - -[dev-dependencies] -tokio.workspace = true diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index 8a841c22036..4a89cdbe2b8 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -7,5 +7,3 @@ extern crate std; #[cfg(feature = "sim")] pub mod sim; - -pub mod io; diff --git a/crates/runtime-core/src/sim/mod.rs b/crates/runtime-core/src/sim/mod.rs index 1a5a53a29bf..e2c231828a1 100644 --- a/crates/runtime-core/src/sim/mod.rs +++ b/crates/runtime-core/src/sim/mod.rs @@ -1,6 +1,5 @@ pub mod buggify; mod executor; -pub mod io; mod rng; pub mod time; diff --git a/crates/runtime-io/Cargo.toml b/crates/runtime-io/Cargo.toml new file mode 100644 index 00000000000..9c94bd86934 --- /dev/null +++ b/crates/runtime-io/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spacetimedb-runtime-io" +version.workspace = true +edition.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +alloc = [] +sim = ["alloc", "dep:futures-channel", "dep:slab", "dep:spin"] + +[dependencies] +futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true} +slab = { version = "0.4", default-features = false, optional = true } +spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } +thiserror = { version = "2.0", default-features = false } +zerocopy = "0.8" + +[dev-dependencies] +spacetimedb-runtime-core = { workspace = true, features = ["sim"] } +spacetimedb-runtime-io = { path = ".", features = ["sim"] } +tokio.workspace = true diff --git a/crates/runtime-core/src/io/buf.rs b/crates/runtime-io/src/buf.rs similarity index 97% rename from crates/runtime-core/src/io/buf.rs rename to crates/runtime-io/src/buf.rs index 35388c62a25..8a537e34326 100644 --- a/crates/runtime-core/src/io/buf.rs +++ b/crates/runtime-io/src/buf.rs @@ -1,6 +1,6 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; -use crate::io::SECTOR_SIZE; +use crate::SECTOR_SIZE; /// Types that can be safely converted to and from sector-aligned byte slices. pub trait AlignedBytes: Sized { @@ -58,12 +58,12 @@ impl AlignedBytes for T { } } -#[cfg(feature = "alloc")] +#[cfg(any(test, feature = "alloc"))] mod boxed { use alloc::boxed::Box; use core::{alloc::Layout, any::TypeId, ptr::NonNull}; - use crate::io::AlignedBytes; + use super::AlignedBytes; /// A type-erased [AlignedBytes] heap allocation. #[derive(Debug)] @@ -173,5 +173,5 @@ mod boxed { } } } -#[cfg(feature = "alloc")] +#[cfg(any(test, feature = "alloc"))] pub use boxed::ErasedBox; diff --git a/crates/runtime-core/src/io/error.rs b/crates/runtime-io/src/error.rs similarity index 100% rename from crates/runtime-core/src/io/error.rs rename to crates/runtime-io/src/error.rs diff --git a/crates/runtime-core/src/io/mod.rs b/crates/runtime-io/src/lib.rs similarity index 96% rename from crates/runtime-core/src/io/mod.rs rename to crates/runtime-io/src/lib.rs index 76979ba9e79..74cea9a6d53 100644 --- a/crates/runtime-core/src/io/mod.rs +++ b/crates/runtime-io/src/lib.rs @@ -1,11 +1,19 @@ +#![no_std] + +#[cfg(any(test, feature = "alloc"))] +extern crate alloc; + mod buf; pub use buf::AlignedBytes; -#[cfg(feature = "alloc")] +#[cfg(any(test, feature = "alloc"))] pub use buf::ErasedBox; mod error; pub use error::ErrorWith; +#[cfg(feature = "sim")] +pub mod sim; + /// Size in bytes of a disk sector. pub const SECTOR_SIZE: usize = 4096; diff --git a/crates/runtime-core/src/sim/io/executor.rs b/crates/runtime-io/src/sim/executor.rs similarity index 91% rename from crates/runtime-core/src/sim/io/executor.rs rename to crates/runtime-io/src/sim/executor.rs index ead10376542..a8c87b9a519 100644 --- a/crates/runtime-core/src/sim/io/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,5 +1,3 @@ -#![allow(unused)] - use alloc::{ boxed::Box, collections::{btree_map, BTreeMap, VecDeque}, @@ -9,19 +7,27 @@ use core::{mem, num::NonZeroUsize, result::Result}; use slab::Slab; use crate::{ - io::{ErasedBox, Statx, SECTOR_SIZE}, - sim::{ - io::{fs, Error, Instant}, - Rng, - }, + sim::{fs, Error}, + ErasedBox, Statx, SECTOR_SIZE, }; -pub use crate::sim::io::fs::Datasync; +pub use crate::sim::fs::Datasync; mod sqe; use sqe::SqeInner; pub use sqe::{LinkKind, Sqe, SqeId}; +pub trait TaskSelector { + /// Deterministically select zero or more tasks to advance. + /// + /// `task_count` is the number of currently outstanding tasks. The returned + /// iterator must return indexes in the range `0..task_count` and not yield + /// duplicate elements. + /// + /// Called once per [Executor::tick]. + fn select_tasks(&self, task_count: usize) -> impl IntoIterator; +} + // TODO: There is no difference between fsync and fdatasync as long as we don't // have an API to fsync the directory of a file after it was created. #[derive(Clone, Copy)] @@ -234,45 +240,44 @@ impl EitherOrBoth { } pub trait FaultInjector { - fn inject_write_sector_fault(&mut self, sqe: &InFlight, op: WriteSector) -> Fault { + fn inject_write_sector_fault(&mut self, _: &InFlight, op: WriteSector) -> Fault { Fault::Visible(Effect::Run(op)) } - fn inject_read_sector_fault(&mut self, sqe: &InFlight, op: ReadSector) -> Fault { + fn inject_read_sector_fault(&mut self, _: &InFlight, op: ReadSector) -> Fault { Fault::Visible(Effect::Run(op)) } - fn inject_open_fault(&mut self, sqe: &InFlight) -> Fault<()> { + fn inject_open_fault(&mut self, _: &InFlight) -> Fault<()> { Fault::Visible(Effect::Run(())) } - fn inject_create_fault(&mut self, sqe: &InFlight) -> Fault<()> { + fn inject_create_fault(&mut self, _: &InFlight) -> Fault<()> { Fault::Visible(Effect::Run(())) } - fn inject_stat_fault(&mut self, sqe: &InFlight) -> Fault<()> { + fn inject_stat_fault(&mut self, _: &InFlight) -> Fault<()> { Fault::Visible(Effect::Run(())) } - fn inject_fallocate_fault(&mut self, sqe: &InFlight) -> Fault<()> { + fn inject_fallocate_fault(&mut self, _: &InFlight) -> Fault<()> { Fault::Visible(Effect::Run(())) } - fn inject_fsync_fault(&mut self, sqe: &InFlight, op: FsyncEffect) -> Fault { + fn inject_fsync_fault(&mut self, _: &InFlight, op: FsyncEffect) -> Fault { Fault::Visible(Effect::Run(op)) } - fn inject_fdatasync_fault(&mut self, sqe: &InFlight, op: Datasync) -> Fault { + fn inject_fdatasync_fault(&mut self, _: &InFlight, op: Datasync) -> Fault { Fault::Visible(Effect::Run(op)) } - fn inject_noop_fault(&mut self, sqe: &InFlight) -> Fault<()> { + fn inject_noop_fault(&mut self, _: &InFlight) -> Fault<()> { Fault::Visible(Effect::Run(())) } } -pub struct NoFaults; -impl FaultInjector for NoFaults {} +impl FaultInjector for () {} /// Completion queue overflow policy. /// @@ -388,7 +393,7 @@ impl Executor { self.cq_overflow = OnCqOverflow::Drop; let executing = mem::take(&mut self.executing); for op in executing { - self.execute(op, faults); + self.execute_op(op, faults); } self.completions.clear(); self.cq_overflow = cq_overflow_orig; @@ -427,6 +432,7 @@ impl Executor { /// over the lifetime of this executor. /// /// Always zero if the executor was configured with [OnCqOverflow::Panic]. + #[allow(unused)] pub fn dropped_completions(&self) -> usize { self.cq_dropped } @@ -440,9 +446,9 @@ impl Executor { /// /// The operation to advance is chosen randomly using `rng`. /// The operation is subject to `faults`. - pub fn tick(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { + pub fn tick(&mut self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut progress = self.schedule(); - progress |= self.execute_random(rng, faults); + progress |= self.execute(task_selector, faults); progress } @@ -482,22 +488,19 @@ impl Executor { progress } - fn execute_random(&mut self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { - if self.executing.is_empty() { - return false; - } - let index = rng.index(self.executing.len()); - if let Some(op) = self.executing.remove(index) { - if let Some(delay) = self.execute(op, faults) { + fn execute(&mut self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { + let mut progress = false; + for index in task_selector.select_tasks(self.executing.len()) { + let op = self.executing.remove(index).expect("task index out of bounds"); + if let Some(delay) = self.execute_op(op, faults) { self.executing.insert(index, delay); } - true - } else { - false + progress |= true } + progress } - fn execute(&mut self, op: Executing, faults: &mut impl FaultInjector) -> Option { + fn execute_op(&mut self, op: Executing, faults: &mut impl FaultInjector) -> Option { op.traverse(|sqe, op| { let in_flight = self.in_flight.get(sqe.key()).expect("invalid sqe id"); match op { @@ -555,10 +558,10 @@ impl Executor { else { unreachable!("invalid sqe: expected write") }; - let mut run = |WriteSector { - page_offset, - buf_offset, - }| { + let run = |WriteSector { + page_offset, + buf_offset, + }| { let bytes = buf.as_bytes(); let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); @@ -611,10 +614,10 @@ impl Executor { else { unreachable!("invalid sqe: expected read") }; - let mut run = |ReadSector { - page_offset, - buf_offset, - }| { + let run = |ReadSector { + page_offset, + buf_offset, + }| { let bytes = buf.as_bytes_mut(); let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); diff --git a/crates/runtime-core/src/sim/io/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs similarity index 99% rename from crates/runtime-core/src/sim/io/executor/sqe.rs rename to crates/runtime-io/src/sim/executor/sqe.rs index ca0ae8f9223..345f1e7e88e 100644 --- a/crates/runtime-core/src/sim/io/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -1,12 +1,12 @@ use alloc::{boxed::Box, vec::Vec}; use crate::{ - io::{ErasedBox, SECTOR_SIZE}, - sim::io::{ + sim::{ executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, WriteSector}, fs::{self, Datasync}, Error, }, + ErasedBox, SECTOR_SIZE, }; /// Opaque identifier of a scheduled [Sqe]. @@ -41,11 +41,13 @@ pub struct Sqe { } impl Sqe { + #[allow(unused)] pub fn link(mut self, kind: Option) -> Self { self.link = kind; self } + #[allow(unused)] pub fn is_linked(&self) -> bool { self.link.is_some() } @@ -93,6 +95,7 @@ impl Sqe { Fdatasync { fd }.into() } + #[allow(unused)] pub fn noop() -> Self { SqeInner::Noop.into() } diff --git a/crates/runtime-core/src/sim/io/fs.rs b/crates/runtime-io/src/sim/fs.rs similarity index 100% rename from crates/runtime-core/src/sim/io/fs.rs rename to crates/runtime-io/src/sim/fs.rs diff --git a/crates/runtime-core/src/sim/io/mod.rs b/crates/runtime-io/src/sim/mod.rs similarity index 91% rename from crates/runtime-core/src/sim/io/mod.rs rename to crates/runtime-io/src/sim/mod.rs index 9970a64e101..c61214244b5 100644 --- a/crates/runtime-core/src/sim/io/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -8,10 +8,7 @@ use core::{ use futures_channel::oneshot; use slab::Slab; -use crate::{ - io::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}, - sim::{io::executor::FaultInjector, Rng}, -}; +use crate::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}; mod executor; use executor::{Cqe, Executor, Sqe}; @@ -19,7 +16,10 @@ use executor::{Cqe, Executor, Sqe}; mod fs; pub use fs::File; -pub use crate::io::SECTOR_SIZE; +pub use crate::{ + sim::executor::{FaultInjector, TaskSelector}, + SECTOR_SIZE, +}; /// Simulated clock measurement. /// @@ -60,11 +60,11 @@ pub struct SimulatorIO { } impl SimulatorIO { - pub fn tick(&self, rng: &Rng, faults: &mut impl FaultInjector) -> bool { + pub fn tick(&self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); - let mut progress = executor.tick(rng, faults); + let mut progress = executor.tick(task_selector, faults); for cqe in executor.completed() { let completion = pending.remove(cqe.user_data().unwrap()); match cqe { @@ -152,8 +152,28 @@ impl SimulatorIO { progress } + /// Simulate a power loss event. + /// + /// All submitted and executing operations are cancelled, and files reset to + /// their durable state. Completions that have not been signalled will be + /// dropped, too. pub fn power_loss(&self) { self.inner.executor.lock().power_loss(); + self.inner.pending.lock().clear(); + } + + /// Simulate a restart event, i.e. process crash. + /// + /// Unlike [Self::power_loss], this will drive the currently executing + /// operations to completion, subject to fault injection. + /// + /// Submissions that were not yet scheduled are dropped. The file state + /// remains unchanged. + /// + /// Completions that were not signalled during shutdown are dropped. + pub fn restart(&self, faults: &mut impl FaultInjector) { + self.inner.executor.lock().restart(faults); + self.inner.pending.lock().clear(); } fn submit( @@ -381,10 +401,16 @@ fn reify( #[cfg(test)] mod tests { - use crate::sim::{io::executor::NoFaults, GlobalRng}; + use spacetimedb_runtime_core::sim::Rng; use super::*; + impl TaskSelector for Rng { + fn select_tasks(&self, task_count: usize) -> impl IntoIterator { + (task_count > 0).then(|| self.index(task_count)) + } + } + struct Runtime { rt: tokio::runtime::LocalRuntime, io: SimulatorIO, @@ -398,13 +424,13 @@ mod tests { .build_local(<_>::default()) .unwrap(), io: SimulatorIO::default(), - rng: GlobalRng::new(0), + rng: Rng::new(0), } } fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); - while self.io.tick(&self.rng, &mut NoFaults) {} + while self.io.tick(&self.rng, &mut ()) {} self.rt.block_on(fut).unwrap() } @@ -453,6 +479,7 @@ mod tests { let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); let mut buf = rt .run(|io| io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) + .map_err(ErrorWith::into_err) .unwrap(); buf.clear(); let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index d23741ce139..10c6c6fbd28 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] tokio.workspace = true spacetimedb-runtime-core = { workspace = true } +spacetimedb-runtime-io = { workspace = true } static_assertions = "1.1" [target.'cfg(unix)'.dependencies] diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index c49f9f690b6..5beaa36b09f 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,7 +5,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -use spacetimedb_runtime_core::io::{AlignedBytes, ErrorWith, SpacetimeIO, Statx}; +use spacetimedb_runtime_io::{AlignedBytes, ErrorWith, SpacetimeIO, Statx}; use static_assertions::assert_not_impl_any; use tokio::runtime; From 9074d720194779d4b248937f1e5f23ca5c61fd21 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 7 Sep 2026 13:28:28 +0200 Subject: [PATCH 26/47] Remove unused `Instant` alias --- Cargo.lock | 14 ++++++++++++++ crates/runtime-io/src/sim/mod.rs | 9 --------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3ace157e6e..e8712df4e30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8241,6 +8241,20 @@ dependencies = [ ] [[package]] + name = "spacetimedb-disk-storage" + version = "2.10.0" + dependencies = [ + "arrayvec", + "async-stream", + "blake3", + "ethnum", + "futures-util", + "spacetimedb-runtime-io", + "thiserror 2.0.17", + "zerocopy", + ] + + [[package]] name = "spacetimedb-dst-lib" version = "2.10.0" dependencies = [ diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index c61214244b5..1cf90d5ca35 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -3,7 +3,6 @@ use core::{ pin::Pin, result::Result, task::{Context, Poll}, - time::Duration, }; use futures_channel::oneshot; use slab::Slab; @@ -21,14 +20,6 @@ pub use crate::{ SECTOR_SIZE, }; -/// Simulated clock measurement. -/// -/// In simulated time, an instant is actually a [Duration] since the time -/// instance was instantiated. To avoid confusion, we use the name "instant" to -/// convey that its semantics are that of the standard library type of the same -/// name. -pub type Instant = Duration; - #[derive(Debug, thiserror::Error)] pub enum Error { #[error("file not found")] From e5a2178eed537ff73e60237c4e96abb674383ae3 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 9 Sep 2026 15:34:29 +0200 Subject: [PATCH 27/47] Replace oneshot channel with custom, no-alloc future --- Cargo.lock | 3 +- crates/runtime-io/Cargo.toml | 3 +- crates/runtime-io/src/lib.rs | 6 +- crates/runtime-io/src/sim/completion.rs | 434 ++++++++++++++++++++++++ crates/runtime-io/src/sim/executor.rs | 4 +- crates/runtime-io/src/sim/mod.rs | 255 +++++--------- crates/runtime/src/io/tokio.rs | 8 +- 7 files changed, 527 insertions(+), 186 deletions(-) create mode 100644 crates/runtime-io/src/sim/completion.rs diff --git a/Cargo.lock b/Cargo.lock index e8712df4e30..ffe1cf25575 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8254,7 +8254,7 @@ dependencies = [ "zerocopy", ] - [[package]] +[[package]] name = "spacetimedb-dst-lib" version = "2.10.0" dependencies = [ @@ -8564,7 +8564,6 @@ dependencies = [ name = "spacetimedb-runtime-io" version = "2.10.0" dependencies = [ - "futures-channel", "slab", "spacetimedb-runtime-core", "spacetimedb-runtime-io", diff --git a/crates/runtime-io/Cargo.toml b/crates/runtime-io/Cargo.toml index 9c94bd86934..5df1af63c4d 100644 --- a/crates/runtime-io/Cargo.toml +++ b/crates/runtime-io/Cargo.toml @@ -9,10 +9,9 @@ workspace = true [features] alloc = [] -sim = ["alloc", "dep:futures-channel", "dep:slab", "dep:spin"] +sim = ["alloc", "dep:slab", "dep:spin"] [dependencies] -futures-channel = { version = "0.3", default-features = false, features = ["alloc"], optional = true} slab = { version = "0.4", default-features = false, optional = true } spin = { version = "0.9", default-features = false, features = ["mutex", "spin_mutex"], optional = true } thiserror = { version = "2.0", default-features = false } diff --git a/crates/runtime-io/src/lib.rs b/crates/runtime-io/src/lib.rs index 74cea9a6d53..997bcac2f9e 100644 --- a/crates/runtime-io/src/lib.rs +++ b/crates/runtime-io/src/lib.rs @@ -30,6 +30,10 @@ impl Statx { } } +/// Cancellation result of a [SpacetimeIO::Completion] future. +#[derive(Clone, Copy, Debug)] +pub struct Cancelled; + /// The canonical, low-level I/O API. /// /// Currently only supports file I/O, but eventually all I/O performed by @@ -61,7 +65,7 @@ pub trait SpacetimeIO { /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 type Error: core::error::Error; /// The completion [Future] of all methods in this trait. - type Completion: Future + Unpin; + type Completion: Future> + Unpin; /// Open the file at `path`. fn open_file(&self, path: &str) -> Self::Completion>; diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs new file mode 100644 index 00000000000..5d90a8eff86 --- /dev/null +++ b/crates/runtime-io/src/sim/completion.rs @@ -0,0 +1,434 @@ +use core::{ + convert::identity, + pin::Pin, + task::{Context, Poll, Waker}, +}; + +use alloc::sync::Arc; + +use crate::{ + sim::{fs, Error, SimulatorInner}, + AlignedBytes, Cancelled, ErasedBox, ErrorWith, Statx, +}; + +pub(crate) enum CompletionState { + Pending(Option), + Ready(T), + Abandoned, +} + +impl CompletionState { + pub(crate) fn complete(&mut self, v: T) -> Option { + match self { + Self::Pending(waker) => { + let waker = waker.take(); + *self = CompletionState::Ready(v); + waker + } + Self::Abandoned => None, + Self::Ready(_) => unreachable!("completion completed twice"), + } + } +} + +pub(crate) enum CompletionHandle { + Write(CompletionState>>), + Read(CompletionState>>), + Open(CompletionState>), + Create(CompletionState>), + Stat(CompletionState>), + Fallocate(CompletionState>), + Fsync(CompletionState>), + Fdatasync(CompletionState>), + #[allow(unused)] + Noop(CompletionState>), +} + +impl CompletionHandle { + fn write_state_mut(&mut self) -> &mut CompletionState>> { + match self { + Self::Write(state) => state, + _ => unreachable!(), + } + } + + fn into_write_state(self) -> CompletionState>> { + match self { + Self::Write(state) => state, + _ => unreachable!(), + } + } + + fn read_state_mut(&mut self) -> &mut CompletionState>> { + match self { + Self::Read(state) => state, + _ => unreachable!(), + } + } + + fn into_read_state(self) -> CompletionState>> { + match self { + Self::Read(state) => state, + _ => unreachable!(), + } + } + + fn open_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Open(state) => state, + _ => unreachable!(), + } + } + + fn into_open_state(self) -> CompletionState> { + match self { + Self::Open(state) => state, + _ => unreachable!(), + } + } + + fn create_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Create(state) => state, + _ => unreachable!(), + } + } + + fn into_create_state(self) -> CompletionState> { + match self { + Self::Create(state) => state, + _ => unreachable!(), + } + } + + fn stat_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Stat(state) => state, + _ => unreachable!(), + } + } + + fn into_stat_state(self) -> CompletionState> { + match self { + Self::Stat(state) => state, + _ => unreachable!(), + } + } + + fn fallocate_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Fallocate(state) => state, + _ => unreachable!(), + } + } + + fn into_fallocate_state(self) -> CompletionState> { + match self { + Self::Fallocate(state) => state, + _ => unreachable!(), + } + } + + fn fsync_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Fsync(state) => state, + _ => unreachable!(), + } + } + + fn into_fsync_state(self) -> CompletionState> { + match self { + Self::Fsync(state) => state, + _ => unreachable!(), + } + } + + fn fdatasync_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Fdatasync(state) => state, + _ => unreachable!(), + } + } + + fn into_fdatasync_state(self) -> CompletionState> { + match self { + Self::Fdatasync(state) => state, + _ => unreachable!(), + } + } + + #[allow(unused)] + fn noop_state_mut(&mut self) -> &mut CompletionState> { + match self { + Self::Noop(state) => state, + _ => unreachable!(), + } + } + + #[allow(unused)] + fn into_noop_state(self) -> CompletionState> { + match self { + Self::Noop(state) => state, + _ => unreachable!(), + } + } +} + +pub struct Completion { + sim: Arc, + key: usize, + poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll>, + drop: fn(&SimulatorInner, usize), +} + +impl Completion>> { + pub(super) fn write(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::write_state_mut, + CompletionHandle::into_write_state, + reify, + cx, + ) + }, + drop: |sim, key| drop_completion(sim, key, CompletionHandle::write_state_mut), + } + } + + pub(super) fn read(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::read_state_mut, + CompletionHandle::into_read_state, + reify, + cx, + ) + }, + + drop: |sim, key| drop_completion(sim, key, CompletionHandle::read_state_mut), + } + } +} + +impl Completion> { + pub(super) fn open(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::open_state_mut, + CompletionHandle::into_open_state, + identity, + cx, + ) + }, + + drop: |sim, key| drop_completion(sim, key, CompletionHandle::open_state_mut), + } + } + + pub(super) fn create(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::create_state_mut, + CompletionHandle::into_create_state, + identity, + cx, + ) + }, + + drop: |sim, key| drop_completion(sim, key, CompletionHandle::create_state_mut), + } + } +} + +impl Completion> { + pub(super) fn stat(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::stat_state_mut, + CompletionHandle::into_stat_state, + identity, + cx, + ) + }, + + drop: |sim, key| drop_completion(sim, key, CompletionHandle::stat_state_mut), + } + } +} + +impl Completion> { + pub(super) fn fallocate(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::fallocate_state_mut, + CompletionHandle::into_fallocate_state, + identity, + cx, + ) + }, + drop: |sim, key| drop_completion(sim, key, CompletionHandle::fallocate_state_mut), + } + } + + pub(super) fn fsync(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::fsync_state_mut, + CompletionHandle::into_fsync_state, + identity, + cx, + ) + }, + drop: |sim, key| drop_completion(sim, key, CompletionHandle::fsync_state_mut), + } + } + + pub(super) fn fdatasync(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::fdatasync_state_mut, + CompletionHandle::into_fdatasync_state, + identity, + cx, + ) + }, + drop: |sim, key| drop_completion(sim, key, CompletionHandle::fdatasync_state_mut), + } + } + + #[allow(unused)] + pub(super) fn noop(sim: Arc, key: usize) -> Self { + Self { + sim, + key, + poll: |sim, key, cx| { + poll_completion( + sim, + key, + CompletionHandle::noop_state_mut, + CompletionHandle::into_noop_state, + identity, + cx, + ) + }, + drop: |sim, key| drop_completion(sim, key, CompletionHandle::noop_state_mut), + } + } +} + +impl Drop for Completion { + fn drop(&mut self) { + (self.drop)(&self.sim, self.key) + } +} + +fn drop_completion( + sim: &SimulatorInner, + key: usize, + state_mut: fn(&mut CompletionHandle) -> &mut CompletionState, +) { + let mut pending = sim.pending.lock(); + if let Some(handle) = pending.get_mut(key) { + let state = (state_mut)(handle); + match state { + CompletionState::Ready(_) => { + pending.remove(key); + } + CompletionState::Pending(_) | CompletionState::Abandoned => { + *state = CompletionState::Abandoned; + } + } + } +} + +fn poll_completion( + sim: &SimulatorInner, + key: usize, + state_mut: fn(&mut CompletionHandle) -> &mut CompletionState, + into_state: fn(CompletionHandle) -> CompletionState, + map: fn(S) -> T, + cx: &mut Context<'_>, +) -> Poll> { + let mut pending = sim.pending.lock(); + let state = (state_mut)(&mut pending[key]); + match state { + CompletionState::Pending(waker) => { + if !waker.as_ref().is_some_and(|waker| waker.will_wake(cx.waker())) { + *waker = Some(cx.waker().clone()); + } + + Poll::Pending + } + CompletionState::Ready(_result) => { + let handle = pending.remove(key); + let state = (into_state)(handle); + + match state { + CompletionState::Ready(result) => Poll::Ready(Ok((map)(result))), + _ => unreachable!(), + } + } + CompletionState::Abandoned => Poll::Ready(Err(Cancelled)), + } +} + +impl Future for Completion { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + (this.poll)(&this.sim, this.key, cx) + } +} + +fn reify( + result: Result>, +) -> Result> { + match result { + Ok(erased) => Ok(erased.into_aligned::()), + Err(ErrorWith { error, with }) => Err(ErrorWith { + error, + with: with.into_aligned::(), + }), + } +} diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index a8c87b9a519..f4919cfbca9 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -352,8 +352,8 @@ impl Executor { Self { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), - in_flight: Slab::new(), - executing: VecDeque::new(), + in_flight: Slab::with_capacity(2 * sq_capacity), + executing: VecDeque::with_capacity(2 * sq_capacity), fstree: BTreeMap::new(), cq_overflow, cq_dropped: 0, diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 1cf90d5ca35..d45bfa997a4 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,13 +1,12 @@ use alloc::{boxed::Box, sync::Arc}; -use core::{ - pin::Pin, - result::Result, - task::{Context, Poll}, -}; -use futures_channel::oneshot; +use core::result::Result; use slab::Slab; -use crate::{AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}; +use crate::{sim::completion::CompletionState, AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}; + +mod completion; +pub use completion::Completion; +use completion::CompletionHandle; mod executor; use executor::{Cqe, Executor, Sqe}; @@ -53,14 +52,14 @@ pub struct SimulatorIO { impl SimulatorIO { pub fn tick(&self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); - let mut pending = self.inner.pending.lock(); let mut progress = executor.tick(task_selector, faults); for cqe in executor.completed() { - let completion = pending.remove(cqe.user_data().unwrap()); - match cqe { + let mut pending = self.inner.pending.lock(); + let completion = &mut pending[cqe.user_data().unwrap()]; + let waker = match cqe { Cqe::Write { result, buf, .. } => { - let CompletionHandle::Write { tx } = completion else { + let CompletionHandle::Write(state) = completion else { unreachable!("invalid cqe / completion pairing") }; let result = match result { @@ -74,10 +73,10 @@ impl SimulatorIO { }), Err(error) => Err(ErrorWith { error, with: buf }), }; - let _ = tx.send(result); + state.complete(result) } Cqe::Read { result, buf, .. } => { - let CompletionHandle::Read { tx } = completion else { + let CompletionHandle::Read(state) = completion else { unreachable!("invalid cqe / completion pairing") }; let result = match result { @@ -91,50 +90,56 @@ impl SimulatorIO { }), Err(error) => Err(ErrorWith { error, with: buf }), }; - let _ = tx.send(result); + state.complete(result) } Cqe::Open { result, .. } => { - let CompletionHandle::Open { tx } = completion else { + let CompletionHandle::Open(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Create { result, .. } => { - let CompletionHandle::Create { tx } = completion else { + let CompletionHandle::Create(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Stat { result, .. } => { - let CompletionHandle::Stat { tx } = completion else { + let CompletionHandle::Stat(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Fallocate { result, .. } => { - let CompletionHandle::Fallocate { tx } = completion else { + let CompletionHandle::Fallocate(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Fsync { result, .. } => { - let CompletionHandle::Fsync { tx } = completion else { + let CompletionHandle::Fsync(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Fdatasync { result, .. } => { - let CompletionHandle::Fdatasync { tx } = completion else { + let CompletionHandle::Fdatasync(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } Cqe::Noop { result, .. } => { - let CompletionHandle::Noop { tx } = completion else { + let CompletionHandle::Noop(state) = completion else { unreachable!("invalid cqe / completion pairing") }; - let _ = tx.send(result); + state.complete(result) } + }; + + // Release lock as waking the future will try to acquire it. + drop(pending); + if let Some(waker) = waker { + waker.wake(); } progress |= true; @@ -167,59 +172,52 @@ impl SimulatorIO { self.inner.pending.lock().clear(); } - fn submit( + fn submit( &self, sqe: Sqe, - completion_handle: impl FnOnce(CompletionSender) -> CompletionHandle, - ) -> Completion> { - let (tx, rx) = oneshot::channel(); - + completion: impl FnOnce(Arc, usize) -> Completion, + completion_handle: impl FnOnce(CompletionState>) -> CompletionHandle, + ) -> Completion { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let pending_entry = pending.vacant_entry(); - match executor.submit([sqe.attach(pending_entry.key())]) { - Err(_sqe) => tx - .send(Err(Error::SubmissionQueueOverflow)) - .unwrap_or_else(|_| unreachable!("rx is alive")), - Ok(()) => { - pending_entry.insert(completion_handle(tx)); - } + let mut state = CompletionState::Pending(None); + if let Err(_sqe) = executor.submit([sqe.attach(pending_entry.key())]) { + state = CompletionState::Ready(Err(Error::SubmissionQueueOverflow)); } + let key = pending_entry.key(); + pending_entry.insert(completion_handle(state)); - rx.into() + completion(self.inner.clone(), key) } fn submit_with( &self, sqe: Sqe, - completion_handle: impl FnOnce(CompletionSender>) -> CompletionHandle, + completion: impl FnOnce(Arc, usize) -> Completion>>, + completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, ) -> Completion>> { - let (tx, rx) = oneshot::channel(); - let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let pending_entry = pending.vacant_entry(); - match executor.submit([sqe.attach(pending_entry.key())]) { - Err(mut sqe) => { - let buf = sqe - .next() - .expect("submitted one sqe therefore one must be returned on overflow") - .into_buf() - .expect("sqe must have been buffer-carrying"); - tx.send(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: buf, - })) - .unwrap_or_else(|_| unreachable!("rx is alive")) - } - Ok(()) => { - pending_entry.insert(completion_handle(tx)); - } + let mut state = CompletionState::Pending(None); + if let Err(mut sqe) = executor.submit([sqe.attach(pending_entry.key())]) { + let buf = sqe + .next() + .expect("submitted one sqe therefore one must be returned on overflow") + .into_buf() + .expect("sqe must have been buffer-carrying"); + state = CompletionState::Ready(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: buf, + })); } + let key = pending_entry.key(); + pending_entry.insert(completion_handle(state)); - Completion::mapped(rx, reify) + completion(self.inner.clone(), key) } } @@ -237,106 +235,17 @@ impl Default for SimulatorInner { } } -pub type CompletionReceiver = oneshot::Receiver>; - -#[must_use = "completions must be polled to completion"] -pub struct Completion(CompletionInner); - -impl Completion { - pub fn mapped( - rx: CompletionReceiver>, - map: fn(Result>) -> T, - ) -> Self { - Self(CompletionInner::Mapped { rx, map }) - } -} - -impl From> for Completion { - fn from(rx: oneshot::Receiver) -> Self { - Self(CompletionInner::Direct { rx }) - } -} - -impl Future for Completion { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - Pin::new(&mut this.0).poll(cx) - } -} - -enum CompletionInner { - Direct { - rx: oneshot::Receiver, - }, - Mapped { - rx: CompletionReceiver>, - map: fn(Result>) -> T, - }, -} - -impl Future for CompletionInner { - type Output = T; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - match this { - Self::Direct { rx } => Pin::new(rx) - .poll(cx) - .map(|result| result.expect("lost completion sender")), - Self::Mapped { rx, map } => Pin::new(rx).poll(cx).map(|result| { - let result = result.expect("lost completion sender"); - map(result) - }), - } - } -} - -type CompletionSender = oneshot::Sender>; -enum CompletionHandle { - Write { - tx: CompletionSender>, - }, - Read { - tx: CompletionSender>, - }, - Open { - tx: CompletionSender, - }, - Create { - tx: CompletionSender, - }, - Stat { - tx: CompletionSender, - }, - Fallocate { - tx: CompletionSender<(), Error>, - }, - Fsync { - tx: CompletionSender<(), Error>, - }, - Fdatasync { - tx: CompletionSender<(), Error>, - }, - // TODO: We may use this for timeouts. - #[allow(unused)] - Noop { - tx: CompletionSender<(), Error>, - }, -} - impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; type Completion = Completion; fn open_file(&self, path: &str) -> Self::Completion> { - self.submit(Sqe::open(path), |tx| CompletionHandle::Open { tx }) + self.submit(Sqe::open(path), Completion::open, CompletionHandle::Open) } fn create_file(&self, path: &str) -> Self::Completion> { - self.submit(Sqe::create(path), |tx| CompletionHandle::Create { tx }) + self.submit(Sqe::create(path), Completion::create, CompletionHandle::Create) } fn write_all_at( @@ -345,9 +254,11 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - self.submit_with(Sqe::write(fd, ErasedBox::from_aligned(buf), offset), |tx| { - CompletionHandle::Write { tx } - }) + self.submit_with( + Sqe::write(fd, ErasedBox::from_aligned(buf), offset), + Completion::write, + CompletionHandle::Write, + ) } fn read_exact_at( @@ -356,37 +267,31 @@ impl SpacetimeIO for SimulatorIO { buf: B, offset: u64, ) -> Self::Completion>> { - self.submit_with(Sqe::read(fd, ErasedBox::from_aligned(buf), offset), |tx| { - CompletionHandle::Read { tx } - }) + self.submit_with( + Sqe::read(fd, ErasedBox::from_aligned(buf), offset), + Completion::read, + CompletionHandle::Read, + ) } fn fsync(&self, fd: Self::Fd) -> Self::Completion> { - self.submit(Sqe::fsync(fd), |tx| CompletionHandle::Fsync { tx }) + self.submit(Sqe::fsync(fd), Completion::fsync, CompletionHandle::Fsync) } fn fdatasync(&self, fd: Self::Fd) -> Self::Completion> { - self.submit(Sqe::fdatasync(fd), |tx| CompletionHandle::Fdatasync { tx }) + self.submit(Sqe::fdatasync(fd), Completion::fdatasync, CompletionHandle::Fdatasync) } fn reserve(&self, fd: Self::Fd, total_size: u64) -> Self::Completion> { - self.submit(Sqe::fallocate(fd, total_size), |tx| CompletionHandle::Fallocate { tx }) + self.submit( + Sqe::fallocate(fd, total_size), + Completion::fallocate, + CompletionHandle::Fallocate, + ) } fn statx(&self, fd: Self::Fd) -> Self::Completion> { - self.submit(Sqe::stat(fd), |tx| CompletionHandle::Stat { tx }) - } -} - -fn reify( - result: Result>, -) -> Result> { - match result { - Ok(erased) => Ok(erased.into_aligned::()), - Err(ErrorWith { error, with }) => Err(ErrorWith { - error, - with: with.into_aligned::(), - }), + self.submit(Sqe::stat(fd), Completion::stat, CompletionHandle::Stat) } } @@ -422,7 +327,7 @@ mod tests { fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); while self.io.tick(&self.rng, &mut ()) {} - self.rt.block_on(fut).unwrap() + self.rt.block_on(fut).unwrap().unwrap() } fn power_loss(&self) { diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index 5beaa36b09f..f6f90ef3a8e 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -5,7 +5,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use std::{io, marker::PhantomData, rc::Rc, sync::Arc}; -use spacetimedb_runtime_io::{AlignedBytes, ErrorWith, SpacetimeIO, Statx}; +use spacetimedb_runtime_io::{AlignedBytes, Cancelled, ErrorWith, SpacetimeIO, Statx}; use static_assertions::assert_not_impl_any; use tokio::runtime; @@ -31,19 +31,19 @@ assert_not_impl_any!(TokioIO: Send); pub struct Completion(tokio::task::JoinHandle); impl Future for Completion { - type Output = T; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); match Pin::new(&mut this.0).poll(cx) { Poll::Pending => Poll::Pending, Poll::Ready(result) => match result { - Ok(output) => Poll::Ready(output), + Ok(output) => Poll::Ready(Ok(output)), Err(error) => { if error.is_panic() { panic::resume_unwind(error.into_panic()) } else if error.is_cancelled() { - panic!("I/O task unexpectedly cancelled"); + Poll::Ready(Err(Cancelled)) } else { unreachable!("unexpected I/O task error") } From a84015a700a54ed2d5ce5047b3142446ce24be94 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 08:30:27 +0200 Subject: [PATCH 28/47] Make cancellation impossible, matching io-uring semantics --- crates/runtime-io/src/lib.rs | 6 +- crates/runtime-io/src/sim/completion.rs | 139 +++++++++++++++--------- crates/runtime-io/src/sim/executor.rs | 46 +++++++- crates/runtime-io/src/sim/mod.rs | 116 ++++---------------- 4 files changed, 152 insertions(+), 155 deletions(-) diff --git a/crates/runtime-io/src/lib.rs b/crates/runtime-io/src/lib.rs index 997bcac2f9e..74cea9a6d53 100644 --- a/crates/runtime-io/src/lib.rs +++ b/crates/runtime-io/src/lib.rs @@ -30,10 +30,6 @@ impl Statx { } } -/// Cancellation result of a [SpacetimeIO::Completion] future. -#[derive(Clone, Copy, Debug)] -pub struct Cancelled; - /// The canonical, low-level I/O API. /// /// Currently only supports file I/O, but eventually all I/O performed by @@ -65,7 +61,7 @@ pub trait SpacetimeIO { /// [alloc_io]: https://github.com/rust-lang/rust/issues/154046 type Error: core::error::Error; /// The completion [Future] of all methods in this trait. - type Completion: Future> + Unpin; + type Completion: Future + Unpin; /// Open the file at `path`. fn open_file(&self, path: &str) -> Self::Completion>; diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index 5d90a8eff86..97b8d18bbb6 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -5,32 +5,60 @@ use core::{ }; use alloc::sync::Arc; +use slab::Slab; use crate::{ sim::{fs, Error, SimulatorInner}, - AlignedBytes, Cancelled, ErasedBox, ErrorWith, Statx, + AlignedBytes, ErasedBox, ErrorWith, Statx, }; +pub use slab::VacantEntry; + pub(crate) enum CompletionState { Pending(Option), Ready(T), - Abandoned, } impl CompletionState { - pub(crate) fn complete(&mut self, v: T) -> Option { + fn complete(&mut self, v: T) -> Option { match self { Self::Pending(waker) => { let waker = waker.take(); *self = CompletionState::Ready(v); waker } - Self::Abandoned => None, Self::Ready(_) => unreachable!("completion completed twice"), } } } +#[derive(Default)] +pub(super) struct PendingCompletions { + inner: Slab, +} + +impl PendingCompletions { + pub(super) fn get_mut(&mut self, key: usize) -> Option<&mut CompletionHandle> { + self.inner.get_mut(key) + } + + pub(super) fn clear(&mut self) { + self.inner.clear(); + } + + pub(super) fn vacant_entry(&mut self) -> VacantEntry<'_, CompletionHandle> { + self.inner.vacant_entry() + } + + fn remove(&mut self, key: usize) -> CompletionHandle { + self.inner.remove(key) + } + + fn try_remove(&mut self, key: usize) -> Option { + self.inner.try_remove(key) + } +} + pub(crate) enum CompletionHandle { Write(CompletionState>>), Read(CompletionState>>), @@ -59,6 +87,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_write(&mut self, result: Result>) -> Option { + self.write_state_mut().complete(result) + } + fn read_state_mut(&mut self) -> &mut CompletionState>> { match self { Self::Read(state) => state, @@ -73,6 +105,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_read(&mut self, result: Result>) -> Option { + self.read_state_mut().complete(result) + } + fn open_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Open(state) => state, @@ -87,6 +123,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_open(&mut self, result: Result) -> Option { + self.open_state_mut().complete(result) + } + fn create_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Create(state) => state, @@ -101,6 +141,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_create(&mut self, result: Result) -> Option { + self.create_state_mut().complete(result) + } + fn stat_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Stat(state) => state, @@ -115,6 +159,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_stat(&mut self, result: Result) -> Option { + self.stat_state_mut().complete(result) + } + fn fallocate_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Fallocate(state) => state, @@ -129,6 +177,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_fallocate(&mut self, result: Result<(), Error>) -> Option { + self.fallocate_state_mut().complete(result) + } + fn fsync_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Fsync(state) => state, @@ -143,6 +195,10 @@ impl CompletionHandle { } } + pub(crate) fn complete_fsync(&mut self, result: Result<(), Error>) -> Option { + self.fsync_state_mut().complete(result) + } + fn fdatasync_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Fdatasync(state) => state, @@ -157,7 +213,10 @@ impl CompletionHandle { } } - #[allow(unused)] + pub(crate) fn complete_fdatasync(&mut self, result: Result<(), Error>) -> Option { + self.fdatasync_state_mut().complete(result) + } + fn noop_state_mut(&mut self) -> &mut CompletionState> { match self { Self::Noop(state) => state, @@ -165,20 +224,22 @@ impl CompletionHandle { } } - #[allow(unused)] fn into_noop_state(self) -> CompletionState> { match self { Self::Noop(state) => state, _ => unreachable!(), } } + + pub(crate) fn complete_noop(&mut self, result: Result<(), Error>) -> Option { + self.noop_state_mut().complete(result) + } } pub struct Completion { sim: Arc, key: usize, - poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll>, - drop: fn(&SimulatorInner, usize), + poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll, } impl Completion>> { @@ -196,7 +257,6 @@ impl Completion>> { cx, ) }, - drop: |sim, key| drop_completion(sim, key, CompletionHandle::write_state_mut), } } @@ -214,8 +274,6 @@ impl Completion>> { cx, ) }, - - drop: |sim, key| drop_completion(sim, key, CompletionHandle::read_state_mut), } } } @@ -235,8 +293,6 @@ impl Completion> { cx, ) }, - - drop: |sim, key| drop_completion(sim, key, CompletionHandle::open_state_mut), } } @@ -254,8 +310,6 @@ impl Completion> { cx, ) }, - - drop: |sim, key| drop_completion(sim, key, CompletionHandle::create_state_mut), } } } @@ -275,8 +329,6 @@ impl Completion> { cx, ) }, - - drop: |sim, key| drop_completion(sim, key, CompletionHandle::stat_state_mut), } } } @@ -296,7 +348,6 @@ impl Completion> { cx, ) }, - drop: |sim, key| drop_completion(sim, key, CompletionHandle::fallocate_state_mut), } } @@ -314,7 +365,6 @@ impl Completion> { cx, ) }, - drop: |sim, key| drop_completion(sim, key, CompletionHandle::fsync_state_mut), } } @@ -332,7 +382,6 @@ impl Completion> { cx, ) }, - drop: |sim, key| drop_completion(sim, key, CompletionHandle::fdatasync_state_mut), } } @@ -351,33 +400,17 @@ impl Completion> { cx, ) }, - drop: |sim, key| drop_completion(sim, key, CompletionHandle::noop_state_mut), } } } +/// Dropping a [Completion] future removes the [CompletionHandle] from the +/// pending list, if it is present. +/// +/// If it is not present, then the future was polled to completion already. impl Drop for Completion { fn drop(&mut self) { - (self.drop)(&self.sim, self.key) - } -} - -fn drop_completion( - sim: &SimulatorInner, - key: usize, - state_mut: fn(&mut CompletionHandle) -> &mut CompletionState, -) { - let mut pending = sim.pending.lock(); - if let Some(handle) = pending.get_mut(key) { - let state = (state_mut)(handle); - match state { - CompletionState::Ready(_) => { - pending.remove(key); - } - CompletionState::Pending(_) | CompletionState::Abandoned => { - *state = CompletionState::Abandoned; - } - } + self.sim.pending.lock().try_remove(self.key); } } @@ -388,32 +421,32 @@ fn poll_completion( into_state: fn(CompletionHandle) -> CompletionState, map: fn(S) -> T, cx: &mut Context<'_>, -) -> Poll> { +) -> Poll { let mut pending = sim.pending.lock(); - let state = (state_mut)(&mut pending[key]); - match state { - CompletionState::Pending(waker) => { - if !waker.as_ref().is_some_and(|waker| waker.will_wake(cx.waker())) { - *waker = Some(cx.waker().clone()); + match pending.get_mut(key) { + None => unreachable!("completion polled after already complete"), + Some(handle) => { + if let CompletionState::Pending(maybe_waker) = (state_mut)(handle) { + if !maybe_waker.as_ref().is_some_and(|waker| waker.will_wake(cx.waker())) { + *maybe_waker = Some(cx.waker().clone()); + } + + return Poll::Pending; } - Poll::Pending - } - CompletionState::Ready(_result) => { let handle = pending.remove(key); let state = (into_state)(handle); match state { - CompletionState::Ready(result) => Poll::Ready(Ok((map)(result))), + CompletionState::Ready(result) => Poll::Ready((map)(result)), _ => unreachable!(), } } - CompletionState::Abandoned => Poll::Ready(Err(Cancelled)), } } impl Future for Completion { - type Output = Result; + type Output = T; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index f4919cfbca9..2ff8849873a 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -3,12 +3,12 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::{mem, num::NonZeroUsize, result::Result}; +use core::{mem, num::NonZeroUsize, result::Result, task::Waker}; use slab::Slab; use crate::{ - sim::{fs, Error}, - ErasedBox, Statx, SECTOR_SIZE, + sim::{completion::CompletionHandle, fs, Error}, + ErasedBox, ErrorWith, Statx, SECTOR_SIZE, }; pub use crate::sim::fs::Datasync; @@ -116,6 +116,46 @@ impl Cqe { | Self::Noop { user_data, .. } => user_data, } } + + pub(crate) fn complete(self, completion: &mut CompletionHandle) -> Option { + match self { + Self::Write { result, buf, .. } => { + let result = match result { + Ok(written) if written == buf.len() => Ok(buf), + Ok(written) => Err(ErrorWith { + error: Error::ShortWrite { + expected: buf.len(), + written, + }, + with: buf, + }), + Err(error) => Err(ErrorWith { error, with: buf }), + }; + completion.complete_write(result) + } + Self::Read { result, buf, .. } => { + let result = match result { + Ok(read) if read == buf.len() => Ok(buf), + Ok(read) => Err(ErrorWith { + error: Error::UnexpectedEof { + expected: buf.len(), + read, + }, + with: buf, + }), + Err(error) => Err(ErrorWith { error, with: buf }), + }; + completion.complete_read(result) + } + Self::Open { result, .. } => completion.complete_open(result), + Self::Create { result, .. } => completion.complete_create(result), + Self::Stat { result, .. } => completion.complete_stat(result), + Self::Fallocate { result, .. } => completion.complete_fallocate(result), + Self::Fsync { result, .. } => completion.complete_fsync(result), + Self::Fdatasync { result, .. } => completion.complete_fdatasync(result), + Self::Noop { result, .. } => completion.complete_noop(result), + } + } } pub struct Blocked { diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index d45bfa997a4..edf695192b6 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,15 +1,17 @@ use alloc::{boxed::Box, sync::Arc}; use core::result::Result; -use slab::Slab; -use crate::{sim::completion::CompletionState, AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx}; +use crate::{ + sim::completion::{CompletionState, PendingCompletions}, + AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx, +}; mod completion; pub use completion::Completion; use completion::CompletionHandle; mod executor; -use executor::{Cqe, Executor, Sqe}; +use executor::{Executor, Sqe}; mod fs; pub use fs::File; @@ -54,96 +56,22 @@ impl SimulatorIO { let mut executor = self.inner.executor.lock(); let mut progress = executor.tick(task_selector, faults); - for cqe in executor.completed() { - let mut pending = self.inner.pending.lock(); - let completion = &mut pending[cqe.user_data().unwrap()]; - let waker = match cqe { - Cqe::Write { result, buf, .. } => { - let CompletionHandle::Write(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - let result = match result { - Ok(written) if written == buf.len() => Ok(buf), - Ok(written) => Err(ErrorWith { - error: Error::ShortWrite { - expected: buf.len(), - written, - }, - with: buf, - }), - Err(error) => Err(ErrorWith { error, with: buf }), - }; - state.complete(result) - } - Cqe::Read { result, buf, .. } => { - let CompletionHandle::Read(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - let result = match result { - Ok(read) if read == buf.len() => Ok(buf), - Ok(read) => Err(ErrorWith { - error: Error::UnexpectedEof { - expected: buf.len(), - read, - }, - with: buf, - }), - Err(error) => Err(ErrorWith { error, with: buf }), - }; - state.complete(result) - } - Cqe::Open { result, .. } => { - let CompletionHandle::Open(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - Cqe::Create { result, .. } => { - let CompletionHandle::Create(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - Cqe::Stat { result, .. } => { - let CompletionHandle::Stat(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - Cqe::Fallocate { result, .. } => { - let CompletionHandle::Fallocate(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - Cqe::Fsync { result, .. } => { - let CompletionHandle::Fsync(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) + executor + .completed() + .map(|cqe| { + let key = cqe.user_data().expect("user data must be set"); + let mut pending = self.inner.pending.lock(); + // If the handle is no longer present in `pending`, the + // completion future was dropped. + let handle = pending.get_mut(key)?; + cqe.complete(handle) + }) + .for_each(|waker| { + if let Some(waker) = waker { + waker.wake(); } - Cqe::Fdatasync { result, .. } => { - let CompletionHandle::Fdatasync(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - Cqe::Noop { result, .. } => { - let CompletionHandle::Noop(state) = completion else { - unreachable!("invalid cqe / completion pairing") - }; - state.complete(result) - } - }; - - // Release lock as waking the future will try to acquire it. - drop(pending); - if let Some(waker) = waker { - waker.wake(); - } - - progress |= true; - } + progress |= true + }); progress } @@ -223,7 +151,7 @@ impl SimulatorIO { struct SimulatorInner { executor: spin::Mutex>, - pending: spin::Mutex>, + pending: spin::Mutex, } impl Default for SimulatorInner { @@ -327,7 +255,7 @@ mod tests { fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); while self.io.tick(&self.rng, &mut ()) {} - self.rt.block_on(fut).unwrap().unwrap() + self.rt.block_on(fut).unwrap() } fn power_loss(&self) { From a3335eaa7cd9da8691f15b9f73cfc4d86a8c1ab3 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 09:46:21 +0200 Subject: [PATCH 29/47] Avoid allocations for results and ops --- crates/runtime-io/src/sim/executor.rs | 146 ++++++++-------- crates/runtime-io/src/sim/executor/sqe.rs | 196 ++++++++++------------ 2 files changed, 166 insertions(+), 176 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 2ff8849873a..885c7d3a878 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,7 +1,6 @@ use alloc::{ boxed::Box, collections::{btree_map, BTreeMap, VecDeque}, - vec::Vec, }; use core::{mem, num::NonZeroUsize, result::Result, task::Waker}; use slab::Slab; @@ -170,39 +169,70 @@ pub struct InFlight { pub user_data: Option, } +pub trait ResultAcc { + fn empty() -> Self; + fn add(&mut self, other: Self); +} + +impl ResultAcc for usize { + fn empty() -> Self { + 0 + } + + fn add(&mut self, other: Self) { + *self += other + } +} + +impl ResultAcc for () { + fn empty() -> Self {} + fn add(&mut self, _: Self) {} +} + +pub struct Results { + remaining: usize, + value: T, + error: Option, +} + +impl Results { + fn new(op_count: usize) -> Self { + Self { + remaining: op_count, + value: T::empty(), + error: None, + } + } + + fn push(&mut self, res: Result) { + match res { + Ok(value) => self.value.add(value), + Err(e) => { + self.error.get_or_insert(e); + } + } + self.remaining -= 1; + } + + fn into_result(self) -> Result { + assert_eq!(self.remaining, 0); + self.error.map_or(Ok(self.value), Err) + } + + fn is_complete(&self) -> bool { + self.remaining == 0 + } +} + pub enum InFlightInner { - Write { - sqe: sqe::Write, - op_count: usize, - results: Vec>, - }, - Read { - sqe: sqe::Read, - op_count: usize, - results: Vec>, - }, - Open { - sqe: sqe::Open, - }, - Create { - sqe: sqe::Create, - }, - Stat { - sqe: sqe::Stat, - }, - Fallocate { - sqe: sqe::Fallocate, - }, - Fsync { - sqe: sqe::Fsync, - op_count: usize, - results: Vec>, - }, - Fdatasync { - sqe: sqe::Fdatasync, - op_count: usize, - results: Vec>, - }, + Write { sqe: sqe::Write, results: Results }, + Read { sqe: sqe::Read, results: Results }, + Open { sqe: sqe::Open }, + Create { sqe: sqe::Create }, + Stat { sqe: sqe::Stat }, + Fallocate { sqe: sqe::Fallocate }, + Fsync { sqe: sqe::Fsync, results: Results<()> }, + Fdatasync { sqe: sqe::Fdatasync, results: Results<()> }, Noop, } @@ -220,6 +250,10 @@ impl Executing { pub enum Fault { /// Drop the operation entirely. + /// + /// Note that this is not generally possible in `io-uring`: an SQE always + /// yields a CQE, even if it was cancelled or returned an error. It may be + /// useful occasionally to construct "byzantine" failures. Skip, /// Put the operation back onto the queue for later execution. Delay(T), @@ -514,8 +548,7 @@ impl Executor { } } let slot = self.in_flight.vacant_entry(); - let (in_flight, ops) = sqe.inner.schedule(SqeId(slot.key())); - self.executing.extend(ops); + let in_flight = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing); slot.insert(InFlight { inner: in_flight, blocked: successors, @@ -590,7 +623,6 @@ impl Executor { inner: InFlightInner::Write { sqe: sqe::Write { fd, buf, .. }, - op_count, results, }, .. @@ -609,8 +641,7 @@ impl Executor { fd.write_page(buf, page_offset as _).map_err(Into::into) }; results.push(eff.traverse(run, Err)); - - results.len() == *op_count + results.is_complete() }; if is_complete { @@ -618,7 +649,6 @@ impl Executor { inner: InFlightInner::Write { sqe: sqe::Write { buf, .. }, - op_count, results, }, blocked, @@ -627,13 +657,7 @@ impl Executor { else { unreachable!("invalid sqe: expected write") }; - assert!(results.len() == op_count); - let bytes_written = results.iter().filter_map(|r| r.as_ref().ok()).sum(); - // TODO: Propagate all errors? - let result = match results.into_iter().find_map(Result::err) { - Some(error) => Err(error), - None => Ok(bytes_written), - }; + let result = results.into_result(); let is_success = result.is_ok(); self.complete(Cqe::Write { result, buf, user_data }); self.schedule_linked(sqe, is_success, blocked); @@ -646,7 +670,6 @@ impl Executor { inner: InFlightInner::Read { sqe: sqe::Read { fd, buf, .. }, - op_count, results, }, .. @@ -665,8 +688,7 @@ impl Executor { fd.read_page(buf, page_offset as _).map_err(Into::into) }; results.push(eff.traverse(run, Err)); - - results.len() == *op_count + results.is_complete() }; if is_complete { @@ -674,7 +696,6 @@ impl Executor { inner: InFlightInner::Read { sqe: sqe::Read { buf, .. }, - op_count, results, }, blocked, @@ -683,13 +704,7 @@ impl Executor { else { unreachable!("invalid sqe: expected read") }; - assert!(results.len() == op_count); - let bytes_read = results.iter().filter_map(|r| r.as_ref().ok()).sum(); - // TODO: Propagate all errors? - let result = match results.into_iter().find_map(Result::err) { - Some(error) => Err(error), - None => Ok(bytes_read), - }; + let result = results.into_result(); let is_success = result.is_ok(); self.complete(Cqe::Read { result, buf, user_data }); self.schedule_linked(sqe, is_success, blocked); @@ -778,7 +793,6 @@ impl Executor { inner: InFlightInner::Fsync { sqe: sqe::Fsync { fd }, - op_count, results, }, .. @@ -794,8 +808,7 @@ impl Executor { Err, ); results.push(result); - - results.len() == *op_count + results.is_complete() }; if is_complete { @@ -807,8 +820,7 @@ impl Executor { else { unreachable!("invalid sqe: expected fsync") }; - // TODO: Propagate all errors? - let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let result = results.into_result(); let is_success = result.is_ok(); self.complete(Cqe::Fsync { result, user_data }); self.schedule_linked(sqe, is_success, blocked); @@ -821,7 +833,6 @@ impl Executor { inner: InFlightInner::Fdatasync { sqe: sqe::Fdatasync { fd }, - op_count, results, }, .. @@ -837,8 +848,7 @@ impl Executor { Err, ); results.push(result); - - results.len() == *op_count + results.is_complete() }; if is_complete { @@ -850,8 +860,7 @@ impl Executor { else { unreachable!("invalid sqe: expected fdatasync") }; - // TODO: Propagate all errors? - let result = results.into_iter().find_map(Result::err).map(Err).unwrap_or(Ok(())); + let result = results.into_result(); let is_success = result.is_ok(); self.complete(Cqe::Fdatasync { result, user_data }); self.schedule_linked(sqe, is_success, blocked); @@ -893,8 +902,7 @@ impl Executor { } } (LinkKind::Soft, true) | (LinkKind::Hard, _) => { - let (inner, ops) = next.schedule(sqe); - self.executing.extend(ops); + let inner = next.schedule(sqe, &mut self.executing); let slot = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id"); *slot = InFlight { inner, diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 345f1e7e88e..90858a5adb7 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -1,8 +1,8 @@ -use alloc::{boxed::Box, vec::Vec}; +use alloc::{boxed::Box, collections::vec_deque::VecDeque}; use crate::{ sim::{ - executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, WriteSector}, + executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, Results, WriteSector}, fs::{self, Datasync}, Error, }, @@ -181,7 +181,7 @@ impl SqeInner { } } - pub(super) fn schedule(self, sqe_id: SqeId) -> (InFlightInner, Vec) { + pub(super) fn schedule(self, sqe_id: SqeId, executing: &mut VecDeque) -> InFlightInner { match self { SqeInner::Write(mut sqe) => { let Write { buf, offset, .. } = &mut sqe; @@ -189,23 +189,17 @@ impl SqeInner { let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - let ops = (0..page_count) - .map(|page| Executing { - sqe: sqe_id, - inner: Operation::WriteSector(WriteSector { - page_offset: first_sector + page, - buf_offset: page * SECTOR_SIZE, - }), - }) - .collect::>(); - let op_count = ops.len(); - let write = InFlightInner::Write { + executing.extend((0..page_count).map(|page| Executing { + sqe: sqe_id, + inner: Operation::WriteSector(WriteSector { + page_offset: first_sector + page, + buf_offset: page * SECTOR_SIZE, + }), + })); + InFlightInner::Write { sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (write, ops) + results: Results::new(page_count), + } } SqeInner::Read(mut sqe) => { let Read { buf, offset, .. } = &mut sqe; @@ -213,113 +207,101 @@ impl SqeInner { let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - let ops = (0..page_count) - .map(|page| Executing { - sqe: sqe_id, - inner: Operation::ReadSector(ReadSector { - page_offset: first_sector + page, - buf_offset: page * SECTOR_SIZE, - }), - }) - .collect::>(); - let op_count = ops.len(); - let read = InFlightInner::Read { + executing.extend((0..page_count).map(|page| Executing { + sqe: sqe_id, + inner: Operation::ReadSector(ReadSector { + page_offset: first_sector + page, + buf_offset: page * SECTOR_SIZE, + }), + })); + InFlightInner::Read { sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (read, ops) + results: Results::new(page_count), + } } - SqeInner::Open(sqe) => ( - InFlightInner::Open { sqe }, - alloc::vec![Executing { + SqeInner::Open(sqe) => { + executing.push_back(Executing { sqe: sqe_id, - inner: Operation::Open - }], - ), - SqeInner::Create(sqe) => ( - InFlightInner::Create { sqe }, - alloc::vec![Executing { + inner: Operation::Open, + }); + InFlightInner::Open { sqe } + } + SqeInner::Create(sqe) => { + executing.push_back(Executing { sqe: sqe_id, - inner: Operation::Create - }], - ), - SqeInner::Stat(sqe) => ( - InFlightInner::Stat { sqe }, - alloc::vec![Executing { + inner: Operation::Create, + }); + InFlightInner::Create { sqe } + } + SqeInner::Stat(sqe) => { + executing.push_back(Executing { sqe: sqe_id, - inner: Operation::Stat - }], - ), - SqeInner::Fallocate(sqe) => ( - InFlightInner::Fallocate { sqe }, - alloc::vec![Executing { + inner: Operation::Stat, + }); + InFlightInner::Stat { sqe } + } + SqeInner::Fallocate(sqe) => { + executing.push_back(Executing { sqe: sqe_id, - inner: Operation::Fallocate - }], - ), + inner: Operation::Fallocate, + }); + InFlightInner::Fallocate { sqe } + } SqeInner::Fsync(sqe) => { let Fsync { fd } = &sqe; let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Sector(offset)), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Length), - }, - }]) - .collect::>(); - let op_count = ops.len(); - let in_flight = InFlightInner::Fsync { + executing.extend( + (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Sector(offset)), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Length), + }, + }]), + ); + InFlightInner::Fsync { sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (in_flight, ops) + results: Results::new(1 + sector_count as usize), + } } SqeInner::Fdatasync(sqe) => { let Fdatasync { fd } = &sqe; let sector_count = fd.len() / SECTOR_SIZE as u64; - let ops = (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Sector(offset), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Length, - }, - }]) - .collect::>(); - let op_count = ops.len(); - let in_flight = InFlightInner::Fdatasync { + executing.extend( + (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Sector(offset), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Length, + }, + }]), + ); + InFlightInner::Fdatasync { sqe, - op_count, - results: Vec::with_capacity(op_count), - }; - - (in_flight, ops) + results: Results::new(1 + sector_count as usize), + } } - SqeInner::Noop => ( - InFlightInner::Noop, - alloc::vec![Executing { + SqeInner::Noop => { + executing.push_back(Executing { sqe: sqe_id, - inner: Operation::Noop - }], - ), + inner: Operation::Noop, + }); + InFlightInner::Noop + } } } } From 1db074bd82f4c32681383a3f3e24794b2a568795 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 11:42:38 +0200 Subject: [PATCH 30/47] Simplify types a bit --- crates/runtime-io/src/sim/executor.rs | 189 +++++++++--------- crates/runtime-io/src/sim/executor/sqe.rs | 225 +++++++--------------- 2 files changed, 168 insertions(+), 246 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 885c7d3a878..8201d02fdd6 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -60,63 +60,55 @@ pub struct ReadSector { } #[derive(Debug)] -pub enum Cqe { +pub struct Cqe { + inner: CqeInner, + user_data: Option, +} +impl Cqe { + pub fn user_data(&self) -> &Option { + &self.user_data + } + + pub(crate) fn complete(self, completion: &mut CompletionHandle) -> Option { + self.inner.complete(completion) + } +} + +#[derive(Debug)] +pub enum CqeInner { Write { result: Result, buf: ErasedBox, - user_data: Option, }, Read { result: Result, buf: ErasedBox, - user_data: Option, }, Open { result: Result, - user_data: Option, }, Create { result: Result, - user_data: Option, }, Stat { result: Result, - user_data: Option, }, Fallocate { result: Result<(), Error>, - user_data: Option, }, Fsync { result: Result<(), Error>, - user_data: Option, }, Fdatasync { result: Result<(), Error>, - user_data: Option, }, Noop { result: Result<(), Error>, - user_data: Option, }, } -impl Cqe { - pub fn user_data(&self) -> &Option { - match self { - Self::Write { user_data, .. } - | Self::Read { user_data, .. } - | Self::Open { user_data, .. } - | Self::Create { user_data, .. } - | Self::Stat { user_data, .. } - | Self::Fallocate { user_data, .. } - | Self::Fsync { user_data, .. } - | Self::Fdatasync { user_data, .. } - | Self::Noop { user_data, .. } => user_data, - } - } - - pub(crate) fn complete(self, completion: &mut CompletionHandle) -> Option { +impl CqeInner { + fn complete(self, completion: &mut CompletionHandle) -> Option { match self { Self::Write { result, buf, .. } => { let result = match result { @@ -164,7 +156,8 @@ pub struct Blocked { } pub struct InFlight { - pub inner: InFlightInner, + pub sqe: SqeInner, + pub pending: Pending, pub blocked: VecDeque>, pub user_data: Option, } @@ -224,16 +217,10 @@ impl Results { } } -pub enum InFlightInner { - Write { sqe: sqe::Write, results: Results }, - Read { sqe: sqe::Read, results: Results }, - Open { sqe: sqe::Open }, - Create { sqe: sqe::Create }, - Stat { sqe: sqe::Stat }, - Fallocate { sqe: sqe::Fallocate }, - Fsync { sqe: sqe::Fsync, results: Results<()> }, - Fdatasync { sqe: sqe::Fdatasync, results: Results<()> }, - Noop, +pub enum Pending { + OneOff, + ReadWrite { results: Results }, + Sync { results: Results<()> }, } struct Executing { @@ -529,7 +516,7 @@ impl Executor { fn schedule(&mut self) -> bool { let mut progress = false; - while let Some(sqe) = self.submissions.pop_front() { + while let Some(mut sqe) = self.submissions.pop_front() { // If the sqe is linked, pop the whole chain. // Links of sqes not submitted in the same batch are ignored. let mut successors = VecDeque::new(); @@ -548,9 +535,10 @@ impl Executor { } } let slot = self.in_flight.vacant_entry(); - let in_flight = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing); + let pending = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing); slot.insert(InFlight { - inner: in_flight, + sqe: sqe.inner, + pending, blocked: successors, user_data: sqe.user_data, }); @@ -620,11 +608,8 @@ impl Executor { fn execute_write_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { let is_complete = { let InFlight { - inner: - InFlightInner::Write { - sqe: sqe::Write { fd, buf, .. }, - results, - }, + sqe: SqeInner::Write { fd, buf, .. }, + pending: Pending::ReadWrite { results }, .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -646,11 +631,8 @@ impl Executor { if is_complete { let InFlight { - inner: - InFlightInner::Write { - sqe: sqe::Write { buf, .. }, - results, - }, + sqe: SqeInner::Write { buf, .. }, + pending: Pending::ReadWrite { results }, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -659,7 +641,10 @@ impl Executor { }; let result = results.into_result(); let is_success = result.is_ok(); - self.complete(Cqe::Write { result, buf, user_data }); + self.complete(Cqe { + inner: CqeInner::Write { result, buf }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } } @@ -667,11 +652,8 @@ impl Executor { fn execute_read_sector(&mut self, sqe: SqeId, eff: EitherOrBoth) { let is_complete = { let InFlight { - inner: - InFlightInner::Read { - sqe: sqe::Read { fd, buf, .. }, - results, - }, + sqe: SqeInner::Read { fd, buf, .. }, + pending: Pending::ReadWrite { results }, .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -693,11 +675,8 @@ impl Executor { if is_complete { let InFlight { - inner: - InFlightInner::Read { - sqe: sqe::Read { buf, .. }, - results, - }, + sqe: SqeInner::Read { buf, .. }, + pending: Pending::ReadWrite { results }, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -706,16 +685,18 @@ impl Executor { }; let result = results.into_result(); let is_success = result.is_ok(); - self.complete(Cqe::Read { result, buf, user_data }); + self.complete(Cqe { + inner: CqeInner::Read { result, buf }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } } fn execute_open(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { - inner: InFlightInner::Open { - sqe: sqe::Open { path }, - }, + sqe: SqeInner::Open { path }, + pending: Pending::OneOff, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -728,15 +709,17 @@ impl Executor { ); let is_success = result.is_ok(); - self.complete(Cqe::Open { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Open { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } fn execute_create(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { - inner: InFlightInner::Create { - sqe: sqe::Create { path }, - }, + sqe: SqeInner::Create { path }, + pending: Pending::OneOff, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -751,13 +734,17 @@ impl Executor { }; let result = eff.traverse(run, Err); let is_success = result.is_ok(); - self.complete(Cqe::Create { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Create { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } fn execute_stat(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { - inner: InFlightInner::Stat { sqe: sqe::Stat { fd } }, + sqe: SqeInner::Stat { fd }, + pending: Pending::OneOff, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -766,15 +753,17 @@ impl Executor { }; let result = eff.traverse(|()| Ok(Statx { size: fd.len() }), Err); let is_success = result.is_ok(); - self.complete(Cqe::Stat { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Stat { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } fn execute_fallocate(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { - inner: InFlightInner::Fallocate { - sqe: sqe::Fallocate { fd, total_len }, - }, + sqe: SqeInner::Fallocate { fd, total_len }, + pending: Pending::OneOff, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -783,18 +772,18 @@ impl Executor { }; let result = eff.traverse(|()| fd.set_len(total_len).map_err(Into::into), Err); let is_success = result.is_ok(); - self.complete(Cqe::Fallocate { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Fallocate { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } fn execute_fsync(&mut self, sqe: SqeId, eff: EitherOrBoth) { let is_complete = { let InFlight { - inner: - InFlightInner::Fsync { - sqe: sqe::Fsync { fd }, - results, - }, + sqe: SqeInner::Fsync { fd }, + pending: Pending::Sync { results }, .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -813,16 +802,20 @@ impl Executor { if is_complete { let InFlight { - inner: InFlightInner::Fsync { results, .. }, + pending: Pending::Sync { results }, blocked, user_data, + .. } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected fsync") }; let result = results.into_result(); let is_success = result.is_ok(); - self.complete(Cqe::Fsync { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Fsync { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } } @@ -830,11 +823,8 @@ impl Executor { fn execute_fdatasync(&mut self, sqe: SqeId, eff: EitherOrBoth) { let is_complete = { let InFlight { - inner: - InFlightInner::Fdatasync { - sqe: sqe::Fdatasync { fd }, - results, - }, + sqe: SqeInner::Fdatasync { fd }, + pending: Pending::Sync { results }, .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -853,23 +843,28 @@ impl Executor { if is_complete { let InFlight { - inner: InFlightInner::Fdatasync { results, .. }, + pending: Pending::Sync { results }, blocked, user_data, + .. } = self.in_flight.remove(sqe.key()) else { unreachable!("invalid sqe: expected fdatasync") }; let result = results.into_result(); let is_success = result.is_ok(); - self.complete(Cqe::Fdatasync { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Fdatasync { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } } fn execute_noop(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { - inner: InFlightInner::Noop, + sqe: SqeInner::Noop, + pending: Pending::OneOff, blocked, user_data, } = self.in_flight.remove(sqe.key()) @@ -878,14 +873,17 @@ impl Executor { }; let result = eff.traverse(Ok, Err); let is_success = result.is_ok(); - self.complete(Cqe::Noop { result, user_data }); + self.complete(Cqe { + inner: CqeInner::Noop { result }, + user_data, + }); self.schedule_linked(sqe, is_success, blocked); } fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { if let Some(Blocked { link, - sqe: next, + sqe: mut next, user_data, }) = blocked.pop_front() { @@ -902,10 +900,11 @@ impl Executor { } } (LinkKind::Soft, true) | (LinkKind::Hard, _) => { - let inner = next.schedule(sqe, &mut self.executing); + let pending = next.schedule(sqe, &mut self.executing); let slot = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id"); *slot = InFlight { - inner, + sqe: next, + pending, blocked, user_data, }; diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 90858a5adb7..9b3a74affe9 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -2,7 +2,7 @@ use alloc::{boxed::Box, collections::vec_deque::VecDeque}; use crate::{ sim::{ - executor::{Cqe, Executing, FsyncEffect, InFlightInner, Operation, ReadSector, Results, WriteSector}, + executor::{Cqe, CqeInner, Executing, FsyncEffect, Operation, Pending, ReadSector, Results, WriteSector}, fs::{self, Datasync}, Error, }, @@ -58,41 +58,41 @@ impl Sqe { } pub fn write(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - Write { fd, buf, offset }.into() + SqeInner::Write { fd, buf, offset }.into() } pub fn read(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - Read { fd, buf, offset }.into() + SqeInner::Read { fd, buf, offset }.into() } pub fn open(path: impl AsRef) -> Self { - Open { + SqeInner::Open { path: path.as_ref().into(), } .into() } pub fn create(path: impl AsRef) -> Self { - Create { + SqeInner::Create { path: path.as_ref().into(), } .into() } pub fn stat(fd: fs::File) -> Self { - Stat { fd }.into() + SqeInner::Stat { fd }.into() } pub fn fallocate(fd: fs::File, len: u64) -> Self { - Fallocate { fd, total_len: len }.into() + SqeInner::Fallocate { fd, total_len: len }.into() } pub fn fsync(fd: fs::File) -> Self { - Fsync { fd }.into() + SqeInner::Fsync { fd }.into() } pub fn fdatasync(fd: fs::File) -> Self { - Fdatasync { fd }.into() + SqeInner::Fdatasync { fd }.into() } #[allow(unused)] @@ -103,7 +103,7 @@ impl Sqe { /// Extract the [ErasedBox] buffer if the [Sqe] carries one. pub(crate) fn into_buf(self) -> Option { match self.inner { - SqeInner::Write(Write { buf, .. }) | SqeInner::Read(Read { buf, .. }) => Some(buf), + SqeInner::Write { buf, .. } | SqeInner::Read { buf, .. } => Some(buf), SqeInner::Open { .. } | SqeInner::Create { .. } | SqeInner::Stat { .. } @@ -126,65 +126,82 @@ impl> From for Sqe { } pub enum SqeInner { - Write(Write), - Read(Read), - Open(Open), - Create(Create), - Stat(Stat), - Fallocate(Fallocate), - Fsync(Fsync), - Fdatasync(Fdatasync), + Write { fd: fs::File, buf: ErasedBox, offset: u64 }, + Read { fd: fs::File, buf: ErasedBox, offset: u64 }, + Open { path: Box }, + Create { path: Box }, + Stat { fd: fs::File }, + Fallocate { fd: fs::File, total_len: u64 }, + Fsync { fd: fs::File }, + Fdatasync { fd: fs::File }, Noop, } impl SqeInner { pub(super) fn cancel(self, user_data: Option) -> Cqe { match self { - SqeInner::Write(Write { buf, .. }) => Cqe::Write { - result: Err(Error::Cancelled), - buf, + SqeInner::Write { buf, .. } => Cqe { + inner: CqeInner::Write { + result: Err(Error::Cancelled), + buf, + }, user_data, }, - SqeInner::Read(Read { buf, .. }) => Cqe::Read { - result: Err(Error::Cancelled), - buf, + SqeInner::Read { buf, .. } => Cqe { + inner: CqeInner::Read { + result: Err(Error::Cancelled), + buf, + }, user_data, }, - SqeInner::Open(..) => Cqe::Open { - result: Err(Error::Cancelled), + SqeInner::Open { .. } => Cqe { + inner: CqeInner::Open { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Create(..) => Cqe::Create { - result: Err(Error::Cancelled), + SqeInner::Create { .. } => Cqe { + inner: CqeInner::Create { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Stat(..) => Cqe::Stat { - result: Err(Error::Cancelled), + SqeInner::Stat { .. } => Cqe { + inner: CqeInner::Stat { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Fallocate(..) => Cqe::Fallocate { - result: Err(Error::Cancelled), + SqeInner::Fallocate { .. } => Cqe { + inner: CqeInner::Fallocate { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Fsync(..) => Cqe::Fsync { - result: Err(Error::Cancelled), + SqeInner::Fsync { .. } => Cqe { + inner: CqeInner::Fsync { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Fdatasync(..) => Cqe::Fdatasync { - result: Err(Error::Cancelled), + SqeInner::Fdatasync { .. } => Cqe { + inner: CqeInner::Fdatasync { + result: Err(Error::Cancelled), + }, user_data, }, - SqeInner::Noop => Cqe::Noop { - result: Err(Error::Cancelled), + SqeInner::Noop => Cqe { + inner: CqeInner::Noop { + result: Err(Error::Cancelled), + }, user_data, }, } } - pub(super) fn schedule(self, sqe_id: SqeId, executing: &mut VecDeque) -> InFlightInner { + pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut VecDeque) -> Pending { match self { - SqeInner::Write(mut sqe) => { - let Write { buf, offset, .. } = &mut sqe; + SqeInner::Write { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; @@ -196,13 +213,11 @@ impl SqeInner { buf_offset: page * SECTOR_SIZE, }), })); - InFlightInner::Write { - sqe, + Pending::ReadWrite { results: Results::new(page_count), } } - SqeInner::Read(mut sqe) => { - let Read { buf, offset, .. } = &mut sqe; + SqeInner::Read { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; @@ -214,42 +229,39 @@ impl SqeInner { buf_offset: page * SECTOR_SIZE, }), })); - InFlightInner::Read { - sqe, + Pending::ReadWrite { results: Results::new(page_count), } } - SqeInner::Open(sqe) => { + SqeInner::Open { .. } => { executing.push_back(Executing { sqe: sqe_id, inner: Operation::Open, }); - InFlightInner::Open { sqe } + Pending::OneOff } - SqeInner::Create(sqe) => { + SqeInner::Create { .. } => { executing.push_back(Executing { sqe: sqe_id, inner: Operation::Create, }); - InFlightInner::Create { sqe } + Pending::OneOff } - SqeInner::Stat(sqe) => { + SqeInner::Stat { .. } => { executing.push_back(Executing { sqe: sqe_id, inner: Operation::Stat, }); - InFlightInner::Stat { sqe } + Pending::OneOff } - SqeInner::Fallocate(sqe) => { + SqeInner::Fallocate { .. } => { executing.push_back(Executing { sqe: sqe_id, inner: Operation::Fallocate, }); - InFlightInner::Fallocate { sqe } + Pending::OneOff } - SqeInner::Fsync(sqe) => { - let Fsync { fd } = &sqe; - + SqeInner::Fsync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; executing.extend( (0..sector_count) @@ -266,14 +278,11 @@ impl SqeInner { }, }]), ); - InFlightInner::Fsync { - sqe, + Pending::Sync { results: Results::new(1 + sector_count as usize), } } - SqeInner::Fdatasync(sqe) => { - let Fdatasync { fd } = &sqe; - + SqeInner::Fdatasync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; executing.extend( (0..sector_count) @@ -290,8 +299,7 @@ impl SqeInner { }, }]), ); - InFlightInner::Fdatasync { - sqe, + Pending::Sync { results: Results::new(1 + sector_count as usize), } } @@ -300,93 +308,8 @@ impl SqeInner { sqe: sqe_id, inner: Operation::Noop, }); - InFlightInner::Noop + Pending::OneOff } } } } - -impl From for SqeInner { - fn from(inner: Write) -> Self { - Self::Write(inner) - } -} - -impl From for SqeInner { - fn from(inner: Read) -> Self { - Self::Read(inner) - } -} - -impl From for SqeInner { - fn from(inner: Open) -> Self { - Self::Open(inner) - } -} - -impl From for SqeInner { - fn from(inner: Create) -> Self { - Self::Create(inner) - } -} - -impl From for SqeInner { - fn from(inner: Stat) -> Self { - Self::Stat(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fallocate) -> Self { - Self::Fallocate(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fsync) -> Self { - Self::Fsync(inner) - } -} - -impl From for SqeInner { - fn from(inner: Fdatasync) -> Self { - Self::Fdatasync(inner) - } -} - -pub struct Write { - pub fd: fs::File, - pub buf: ErasedBox, - pub offset: u64, -} - -pub struct Read { - pub fd: fs::File, - pub offset: u64, - pub buf: ErasedBox, -} - -pub struct Open { - pub path: Box, -} - -pub struct Create { - pub path: Box, -} - -pub struct Stat { - pub fd: fs::File, -} - -pub struct Fallocate { - pub fd: fs::File, - pub total_len: u64, -} - -pub struct Fsync { - pub fd: fs::File, -} - -pub struct Fdatasync { - pub fd: fs::File, -} From 7dbb5c5b344c8c42c29fcd903c7bdd92ea086c24 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 13:52:04 +0200 Subject: [PATCH 31/47] Require io buffers to point to stable memory, i.e. Box --- crates/runtime-io/Cargo.toml | 3 +- crates/runtime-io/src/buf.rs | 170 +++++++++++------------- crates/runtime-io/src/lib.rs | 17 +-- crates/runtime-io/src/sim/completion.rs | 6 +- crates/runtime-io/src/sim/mod.rs | 27 ++-- crates/runtime/src/io/tokio.rs | 8 +- 6 files changed, 107 insertions(+), 124 deletions(-) diff --git a/crates/runtime-io/Cargo.toml b/crates/runtime-io/Cargo.toml index 5df1af63c4d..316d30bbdd1 100644 --- a/crates/runtime-io/Cargo.toml +++ b/crates/runtime-io/Cargo.toml @@ -8,8 +8,7 @@ rust-version.workspace = true workspace = true [features] -alloc = [] -sim = ["alloc", "dep:slab", "dep:spin"] +sim = ["dep:slab", "dep:spin"] [dependencies] slab = { version = "0.4", default-features = false, optional = true } diff --git a/crates/runtime-io/src/buf.rs b/crates/runtime-io/src/buf.rs index 8a537e34326..7dbbbd911b7 100644 --- a/crates/runtime-io/src/buf.rs +++ b/crates/runtime-io/src/buf.rs @@ -1,3 +1,5 @@ +use alloc::boxed::Box; +use core::{alloc::Layout, any::TypeId, ptr::NonNull}; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::SECTOR_SIZE; @@ -58,120 +60,100 @@ impl AlignedBytes for T { } } -#[cfg(any(test, feature = "alloc"))] -mod boxed { - use alloc::boxed::Box; - use core::{alloc::Layout, any::TypeId, ptr::NonNull}; - - use super::AlignedBytes; +/// A type-erased [AlignedBytes] heap allocation. +#[derive(Debug)] +pub struct ErasedBox { + ptr: NonNull, + len: usize, + layout: Layout, + ty: TypeId, +} - /// A type-erased [AlignedBytes] heap allocation. - #[derive(Debug)] - pub struct ErasedBox { - ptr: NonNull, - len: usize, - layout: Layout, - ty: TypeId, +impl ErasedBox { + /// Create an [ErasedBox] from boxed [AlignedBytes].. + pub fn from_aligned(b: Box) -> Self { + let () = B::ASSERT_VALID_LAYOUT; + + let ptr = Box::into_raw(b); + Self { + ptr: NonNull::new(ptr.cast()).unwrap(), + len: size_of::(), + layout: Layout::from_size_align(size_of::(), align_of::()).unwrap(), + ty: TypeId::of::(), + } } - impl ErasedBox { - /// Create an [ErasedBox] from `B` by allocating a new [Box]. - pub fn from_aligned(b: B) -> Self { - Self::from_aligned_box(Box::new(b)) - } + /// Reify `B` via casting. + pub fn into_aligned(self) -> Box { + assert_eq!(self.len, size_of::()); + assert_eq!(self.ty, TypeId::of::()); - /// Create an [ErasedBox] from an already-boxed `B`. - pub fn from_aligned_box(b: Box) -> Self { - let () = B::ASSERT_VALID_LAYOUT; - - let ptr = Box::into_raw(b); - Self { - ptr: NonNull::new(ptr.cast()).unwrap(), - len: size_of::(), - layout: Layout::from_size_align(size_of::(), align_of::()).unwrap(), - ty: TypeId::of::(), - } - } + let boxed = unsafe { Box::from_raw(self.ptr.as_ptr().cast::()) }; + // Prevent drop, which would deallocate. + core::mem::forget(self); - /// Reify `B` via casting. - pub fn into_aligned(self) -> B { - *Self::into_aligned_box(self) - } + boxed + } - /// Reify `B` via casting, without unboxing. - pub fn into_aligned_box(self) -> Box { - assert_eq!(self.len, size_of::()); - assert_eq!(self.ty, TypeId::of::()); + pub fn as_mut_ptr(&self) -> *mut u8 { + self.ptr.as_ptr() + } - let boxed = unsafe { Box::from_raw(self.ptr.as_ptr().cast::()) }; - // Prevent drop, which would deallocate. - core::mem::forget(self); + pub fn len(&self) -> usize { + self.len + } - boxed - } + pub fn is_empty(&self) -> bool { + self.len == 0 + } - pub fn as_mut_ptr(&self) -> *mut u8 { - self.ptr.as_ptr() - } + pub fn as_bytes(&self) -> &[u8] { + unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } - pub fn len(&self) -> usize { - self.len - } + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + } +} - pub fn is_empty(&self) -> bool { - self.len == 0 - } +impl Drop for ErasedBox { + fn drop(&mut self) { + unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } + } +} - pub fn as_bytes(&self) -> &[u8] { - unsafe { core::slice::from_raw_parts(self.ptr.as_ptr(), self.len) } - } +#[cfg(test)] +mod tests { + use super::*; + + #[repr(C, align(4096))] + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct Trivial([u8; 4096]); - pub fn as_bytes_mut(&mut self) -> &mut [u8] { - unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len) } + impl AlignedBytes for Trivial { + fn as_bytes(&self) -> &[u8] { + &self.0 } - } - impl Drop for ErasedBox { - fn drop(&mut self) { - unsafe { alloc::alloc::dealloc(self.ptr.as_ptr(), self.layout) } + fn as_bytes_mut(&mut self) -> &mut [u8] { + &mut self.0 } - } - #[cfg(test)] - mod tests { - use super::*; - - #[repr(C, align(4096))] - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - struct Trivial([u8; 4096]); - - impl AlignedBytes for Trivial { - fn as_bytes(&self) -> &[u8] { - &self.0 - } - - fn as_bytes_mut(&mut self) -> &mut [u8] { - &mut self.0 - } - - fn from_bytes(b: &[u8]) -> Self { - assert_eq!(b.len(), size_of::()); - let mut a = [0; 4096]; - a.copy_from_slice(b); - Self(a) - } + fn from_bytes(b: &[u8]) -> Self { + assert_eq!(b.len(), size_of::()); + let mut a = [0; 4096]; + a.copy_from_slice(b); + Self(a) } + } - #[test] - fn roundtrip_preserves_value() { - let t = Trivial([32; 4096]); + #[test] + fn roundtrip_preserves_value() { + let t = Trivial([32; 4096]); - let erased = ErasedBox::from_aligned(t); - let reified = erased.into_aligned::(); + let erased = ErasedBox::from_aligned(Box::new(t)); + let reified = erased.into_aligned::(); - assert_eq!(reified, t); - } + assert_eq!(reified, Box::new(t)); } } -#[cfg(any(test, feature = "alloc"))] -pub use boxed::ErasedBox; diff --git a/crates/runtime-io/src/lib.rs b/crates/runtime-io/src/lib.rs index 74cea9a6d53..8bcc978449d 100644 --- a/crates/runtime-io/src/lib.rs +++ b/crates/runtime-io/src/lib.rs @@ -1,12 +1,11 @@ #![no_std] -#[cfg(any(test, feature = "alloc"))] extern crate alloc; +use alloc::boxed::Box; + mod buf; -pub use buf::AlignedBytes; -#[cfg(any(test, feature = "alloc"))] -pub use buf::ErasedBox; +pub use buf::{AlignedBytes, ErasedBox}; mod error; pub use error::ErrorWith; @@ -30,6 +29,8 @@ impl Statx { } } +pub type ReadWriteResult = Result, ErrorWith>>; + /// The canonical, low-level I/O API. /// /// Currently only supports file I/O, but eventually all I/O performed by @@ -81,9 +82,9 @@ pub trait SpacetimeIO { fn write_all_at( &self, fd: Self::Fd, - buf: B, + buf: Box, offset: u64, - ) -> Self::Completion>>; + ) -> Self::Completion>; /// Read `size_of::()` bytes from `fd` at `offset` and interpret them at /// type `B`. @@ -97,9 +98,9 @@ pub trait SpacetimeIO { fn read_exact_at( &self, fd: Self::Fd, - buf: B, + buf: Box, offset: u64, - ) -> Self::Completion>>; + ) -> Self::Completion>; /// Call `fsync(2)` on `fd`. fn fsync(&self, fd: Self::Fd) -> Self::Completion>; diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index 97b8d18bbb6..31bc11ef92b 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -4,7 +4,7 @@ use core::{ task::{Context, Poll, Waker}, }; -use alloc::sync::Arc; +use alloc::{boxed::Box, sync::Arc}; use slab::Slab; use crate::{ @@ -242,7 +242,7 @@ pub struct Completion { poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll, } -impl Completion>> { +impl Completion, ErrorWith>>> { pub(super) fn write(sim: Arc, key: usize) -> Self { Self { sim, @@ -456,7 +456,7 @@ impl Future for Completion { fn reify( result: Result>, -) -> Result> { +) -> Result, ErrorWith>> { match result { Ok(erased) => Ok(erased.into_aligned::()), Err(ErrorWith { error, with }) => Err(ErrorWith { diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index edf695192b6..20887d47254 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -123,9 +123,9 @@ impl SimulatorIO { fn submit_with( &self, sqe: Sqe, - completion: impl FnOnce(Arc, usize) -> Completion>>, + completion: impl FnOnce(Arc, usize) -> Completion, ErrorWith>>>, completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, - ) -> Completion>> { + ) -> Completion, ErrorWith>>> { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let pending_entry = pending.vacant_entry(); @@ -179,9 +179,9 @@ impl SpacetimeIO for SimulatorIO { fn write_all_at( &self, fd: Self::Fd, - buf: B, + buf: Box, offset: u64, - ) -> Self::Completion>> { + ) -> Self::Completion, ErrorWith>>> { self.submit_with( Sqe::write(fd, ErasedBox::from_aligned(buf), offset), Completion::write, @@ -192,9 +192,9 @@ impl SpacetimeIO for SimulatorIO { fn read_exact_at( &self, fd: Self::Fd, - buf: B, + buf: Box, offset: u64, - ) -> Self::Completion>> { + ) -> Self::Completion, ErrorWith>>> { self.submit_with( Sqe::read(fd, ErasedBox::from_aligned(buf), offset), Completion::read, @@ -301,8 +301,9 @@ mod tests { let rt = Runtime::new(); let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let buf = Box::new(Buf([22; 2 * SECTOR_SIZE])); let mut buf = rt - .run(|io| io.write_all_at(fd.clone(), Buf([22; 2 * SECTOR_SIZE]), 0)) + .run(|io| io.write_all_at(fd.clone(), buf, 0)) .map_err(ErrorWith::into_err) .unwrap(); buf.clear(); @@ -316,8 +317,8 @@ mod tests { let rt = Runtime::new(); let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); - let buf = { - let mut buf = Buf([0; SECTOR_SIZE]); + let buf: Box> = { + let mut buf = Box::new(Buf([0; SECTOR_SIZE])); for i in 0usize..2 { buf.0.fill((i + 1) as u8 * 2); let offset = (i * SECTOR_SIZE) as u64; @@ -344,7 +345,7 @@ mod tests { // Check that reserved space reads as zeroes. let buf = rt - .run(|io| io.read_exact_at(fd.clone(), Buf([1; 2 * SECTOR_SIZE]), 0)) + .run(|io| io.read_exact_at(fd.clone(), Box::new(Buf([1; 2 * SECTOR_SIZE])), 0)) .unwrap(); assert_eq!(buf.0, [0; 2 * SECTOR_SIZE]); @@ -354,7 +355,7 @@ mod tests { // Overwriting the second sector works. let buf = rt - .run(|io| io.write_all_at(fd.clone(), Buf([42; SECTOR_SIZE]), SECTOR_SIZE as u64)) + .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([42; SECTOR_SIZE])), SECTOR_SIZE as u64)) .unwrap(); let buf = rt .run(|io| io.read_exact_at(fd.clone(), buf, SECTOR_SIZE as u64)) @@ -380,7 +381,7 @@ mod tests { let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); let mut buf = rt - .run(|io| io.write_all_at(fd.clone(), Buf([1; SECTOR_SIZE]), 0)) + .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([1; SECTOR_SIZE])), 0)) .map_err(ErrorWith::into_err) .unwrap(); buf.clear(); @@ -388,7 +389,7 @@ mod tests { rt.run(|io| io.fdatasync(fd.clone())).unwrap(); let mut buf = rt - .run(|io| io.write_all_at(fd.clone(), Buf([2; SECTOR_SIZE]), SECTOR_SIZE as u64)) + .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([2; SECTOR_SIZE])), SECTOR_SIZE as u64)) .map_err(ErrorWith::into_err) .unwrap(); buf.clear(); diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index c86aae15f59..f08418af0a6 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -94,9 +94,9 @@ impl SpacetimeIO for TokioIO { fn write_all_at( &self, fd: Self::Fd, - buf: B, + buf: Box, offset: u64, - ) -> Self::Completion>> { + ) -> Self::Completion, ErrorWith>>> { self.rt .spawn_blocking(move || match platform::write_all_at(&fd, buf.as_bytes(), offset) { Ok(()) => Ok(buf), @@ -108,9 +108,9 @@ impl SpacetimeIO for TokioIO { fn read_exact_at( &self, fd: Self::Fd, - mut buf: B, + mut buf: Box, offset: u64, - ) -> Self::Completion>> { + ) -> Self::Completion, ErrorWith>>> { self.rt .spawn_blocking(move || match platform::read_exact_at(&fd, buf.as_bytes_mut(), offset) { Ok(()) => Ok(buf), From 311b7f061669bf88e873a1c692a096ea6d349de5 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 14:10:15 +0200 Subject: [PATCH 32/47] Avoid internal allocation for paths --- crates/runtime-io/src/lib.rs | 4 ++-- crates/runtime-io/src/sim/executor/sqe.rs | 14 ++++------- crates/runtime-io/src/sim/mod.rs | 29 +++++++++++++---------- crates/runtime/src/io/tokio.rs | 19 ++++++++------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/crates/runtime-io/src/lib.rs b/crates/runtime-io/src/lib.rs index 8bcc978449d..bf65f8a2392 100644 --- a/crates/runtime-io/src/lib.rs +++ b/crates/runtime-io/src/lib.rs @@ -65,12 +65,12 @@ pub trait SpacetimeIO { type Completion: Future + Unpin; /// Open the file at `path`. - fn open_file(&self, path: &str) -> Self::Completion>; + fn open_file(&self, path: Box) -> Self::Completion>; /// Create the file at `path`. /// /// Returns an error if the file already exists. - fn create_file(&self, path: &str) -> Self::Completion>; + fn create_file(&self, path: Box) -> Self::Completion>; /// Write `buf` to `fd` at `offset`. /// diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 9b3a74affe9..f39daeef420 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -65,18 +65,12 @@ impl Sqe { SqeInner::Read { fd, buf, offset }.into() } - pub fn open(path: impl AsRef) -> Self { - SqeInner::Open { - path: path.as_ref().into(), - } - .into() + pub fn open(path: Box) -> Self { + SqeInner::Open { path }.into() } - pub fn create(path: impl AsRef) -> Self { - SqeInner::Create { - path: path.as_ref().into(), - } - .into() + pub fn create(path: Box) -> Self { + SqeInner::Create { path }.into() } pub fn stat(fd: fs::File) -> Self { diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 20887d47254..1603be8037a 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -3,7 +3,7 @@ use core::result::Result; use crate::{ sim::completion::{CompletionState, PendingCompletions}, - AlignedBytes, ErasedBox, ErrorWith, SpacetimeIO, Statx, + AlignedBytes, ErasedBox, ErrorWith, ReadWriteResult, SpacetimeIO, Statx, }; mod completion; @@ -123,9 +123,9 @@ impl SimulatorIO { fn submit_with( &self, sqe: Sqe, - completion: impl FnOnce(Arc, usize) -> Completion, ErrorWith>>>, + completion: impl FnOnce(Arc, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, - ) -> Completion, ErrorWith>>> { + ) -> Completion> { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); let pending_entry = pending.vacant_entry(); @@ -168,11 +168,11 @@ impl SpacetimeIO for SimulatorIO { type Error = Error; type Completion = Completion; - fn open_file(&self, path: &str) -> Self::Completion> { + fn open_file(&self, path: Box) -> Self::Completion> { self.submit(Sqe::open(path), Completion::open, CompletionHandle::Open) } - fn create_file(&self, path: &str) -> Self::Completion> { + fn create_file(&self, path: Box) -> Self::Completion> { self.submit(Sqe::create(path), Completion::create, CompletionHandle::Create) } @@ -266,7 +266,7 @@ mod tests { #[test] fn create_file() { let rt = Runtime::new(); - rt.run(|io| io.create_file("/data/test")).unwrap(); + rt.run(|io| io.create_file("/data/test".into())).unwrap(); } #[derive(Debug)] @@ -300,7 +300,7 @@ mod tests { fn write_read_roundtrip() { let rt = Runtime::new(); - let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let fd = rt.run(|io| io.create_file("/data/test".into())).unwrap(); let buf = Box::new(Buf([22; 2 * SECTOR_SIZE])); let mut buf = rt .run(|io| io.write_all_at(fd.clone(), buf, 0)) @@ -316,7 +316,7 @@ mod tests { fn write_read_at_offset() { let rt = Runtime::new(); - let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let fd = rt.run(|io| io.create_file("/data/test".into())).unwrap(); let buf: Box> = { let mut buf = Box::new(Buf([0; SECTOR_SIZE])); for i in 0usize..2 { @@ -340,7 +340,7 @@ mod tests { fn preallocate() { let rt = Runtime::new(); - let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let fd = rt.run(|io| io.create_file("/data/test".into())).unwrap(); rt.run(|io| io.reserve(fd.clone(), 2 * SECTOR_SIZE as u64)).unwrap(); // Check that reserved space reads as zeroes. @@ -370,16 +370,19 @@ mod tests { fn open_succeeds_after_create() { let rt = Runtime::new(); - matches!(rt.run(|io| io.open_file("/data/test")), Err(Error::FileNotFound { .. })); - rt.run(|io| io.create_file("/data/test")).unwrap(); - assert!(rt.run(|io| io.open_file("/data/test")).is_ok()); + matches!( + rt.run(|io| io.open_file("/data/test".into())), + Err(Error::FileNotFound { .. }) + ); + rt.run(|io| io.create_file("/data/test".into())).unwrap(); + assert!(rt.run(|io| io.open_file("/data/test".into())).is_ok()); } #[test] fn unsynced_data_is_lost_after_power_loss() { let rt = Runtime::new(); - let fd = rt.run(|io| io.create_file("/data/test")).unwrap(); + let fd = rt.run(|io| io.create_file("/data/test".into())).unwrap(); let mut buf = rt .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([1; SECTOR_SIZE])), 0)) .map_err(ErrorWith::into_err) diff --git a/crates/runtime/src/io/tokio.rs b/crates/runtime/src/io/tokio.rs index f08418af0a6..1ae1e36fddd 100644 --- a/crates/runtime/src/io/tokio.rs +++ b/crates/runtime/src/io/tokio.rs @@ -69,8 +69,8 @@ impl SpacetimeIO for TokioIO { type Error = io::Error; type Completion = Completion; - fn open_file(&self, path: &str) -> Self::Completion> { - let path = PathBuf::from(path); + fn open_file(&self, path: Box) -> Self::Completion> { + let path = PathBuf::from(&*path); self.rt .spawn_blocking(move || { let mut open_options = std::fs::File::options(); @@ -80,8 +80,8 @@ impl SpacetimeIO for TokioIO { .into() } - fn create_file(&self, path: &str) -> Self::Completion> { - let path = PathBuf::from(path); + fn create_file(&self, path: Box) -> Self::Completion> { + let path = PathBuf::from(&*path); self.rt .spawn_blocking(move || { let mut open_options = std::fs::File::options(); @@ -108,13 +108,16 @@ impl SpacetimeIO for TokioIO { fn read_exact_at( &self, fd: Self::Fd, - mut buf: Box, + buf: Box, offset: u64, ) -> Self::Completion, ErrorWith>>> { self.rt - .spawn_blocking(move || match platform::read_exact_at(&fd, buf.as_bytes_mut(), offset) { - Ok(()) => Ok(buf), - Err(error) => Err(ErrorWith { error, with: buf }), + .spawn_blocking(move || { + let mut buf = buf; + match platform::read_exact_at(&fd, buf.as_bytes_mut(), offset) { + Ok(()) => Ok(buf), + Err(error) => Err(ErrorWith { error, with: buf }), + } }) .into() } From e4221e1e293d8c629eddfaa6fceaf92bc6ef52ce Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 15:11:41 +0200 Subject: [PATCH 33/47] Put file behind a single Arc, and perform copy-on-write when a page is already durable --- crates/runtime-io/src/sim/executor.rs | 2 +- crates/runtime-io/src/sim/fs.rs | 94 +++++++++++++++++++-------- 2 files changed, 68 insertions(+), 28 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 8201d02fdd6..19b74963451 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -727,7 +727,7 @@ impl Executor { unreachable!("invalid sqe: expected create") }; let run = |()| match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::new()).clone()), + btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::default()).clone()), btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { path: entry.key().clone(), }), diff --git a/crates/runtime-io/src/sim/fs.rs b/crates/runtime-io/src/sim/fs.rs index 2b6db7ee2c5..ce589d64931 100644 --- a/crates/runtime-io/src/sim/fs.rs +++ b/crates/runtime-io/src/sim/fs.rs @@ -59,14 +59,28 @@ impl PageMap { } /// Get the page at `index` for reading. Uses the volatile state. - fn get_page(&self, index: PageIndex) -> Option> { + fn readonly_page(&self, index: PageIndex) -> Option> { self.volatile.get(&index).cloned() } /// Get the page at `index` for writing, or allocate a new page. /// Uses the volatile state. - fn get_or_allocate_page(&mut self, index: PageIndex) -> Arc { - Arc::clone(self.volatile.entry(index).or_insert_with(|| Arc::new(Page::zeroed()))) + fn writable_page(&mut self, index: PageIndex) -> Arc { + let page = self.volatile.entry(index).or_insert_with(|| Arc::new(Page::zeroed())); + + // Copy-on-write if the page is in the durable state. + if self + .durable + .get(&index) + .is_some_and(|durable| Arc::ptr_eq(durable, page)) + { + let bytes = *page.bytes.lock(); + *page = Arc::new(Page { + bytes: spin::Mutex::new(bytes), + }); + } + + Arc::clone(page) } /// Change the allocated space, allocating or deallocating pages as needed. @@ -117,45 +131,71 @@ pub enum Datasync { /// /// Read and write operations must be page-aligned. Only full pages can be read /// or written. Writing a page is atomic. -#[derive(Clone)] +#[derive(Clone, Debug, Default)] pub struct File { - pages: Arc>, + inner: Arc, +} + +impl File { + pub fn power_loss(&self) { + self.inner.power_loss(); + } + + pub fn len(&self) -> u64 { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result { + self.inner.read_page(dst, index) + } - volatile_len: Arc, - durable_len: Arc, + pub fn write_page(&self, src: &[u8], index: u64) -> Result { + self.inner.write_page(src, index) + } + + pub fn fdatasync(&self, ops: impl IntoIterator) { + self.inner.fdatasync(ops); + } + + pub fn set_len(&self, new_len: u64) -> Result<()> { + self.inner.set_len(new_len) + } } -impl fmt::Debug for File { +#[derive(Default)] +struct FileInner { + pages: spin::Mutex, + volatile_len: AtomicU64, + durable_len: AtomicU64, +} + +impl fmt::Debug for FileInner { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("File") + f.debug_struct("FileInner") .field("volatile_len", &self.volatile_len) .field("durable_len", &self.durable_len) .finish() } } -impl File { - pub(super) fn new() -> Self { - Self { - pages: <_>::default(), - volatile_len: <_>::default(), - durable_len: <_>::default(), - } - } - +impl FileInner { /// Simulate a crash by resetting to the durable state. - pub(super) fn power_loss(&self) { + fn power_loss(&self) { self.volatile_len .store(self.durable_len.load(Ordering::Relaxed), Ordering::Relaxed); self.pages.lock().power_loss(); } - pub(super) fn len(&self) -> u64 { + fn len(&self) -> u64 { self.volatile_len.load(Ordering::Relaxed) } #[allow(unused)] - pub(super) fn is_empty(&self) -> bool { + fn is_empty(&self) -> bool { self.len() == 0 } @@ -165,7 +205,7 @@ impl File { /// /// Extending allocates pages eagerly as needed. Shrinking drops all pages /// at or beyond the new EOF. - pub(super) fn set_len(&self, new_len: u64) -> Result<()> { + fn set_len(&self, new_len: u64) -> Result<()> { if !new_len.is_multiple_of(PAGE_SIZE_U64) { return Err(Error::UnalignedOffset); } @@ -178,7 +218,7 @@ impl File { } /// Read one complete page. - pub(super) fn read_page(&self, dst: &mut [u8], index: u64) -> Result { + fn read_page(&self, dst: &mut [u8], index: u64) -> Result { if dst.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } @@ -202,7 +242,7 @@ impl File { } /// Write one complete page. - pub(super) fn write_page(&self, src: &[u8], index: u64) -> Result { + fn write_page(&self, src: &[u8], index: u64) -> Result { if src.len() != PAGE_SIZE { return Err(Error::UnalignedBuffer); } @@ -228,7 +268,7 @@ impl File { /// It is the caller's responsibility to decide whether the operation is /// considered successful - a partial operation may report success, or a /// complete operation may report failure. - pub(super) fn fdatasync(&self, ops: impl IntoIterator) { + fn fdatasync(&self, ops: impl IntoIterator) { for op in ops { match op { Datasync::Sector(offset) => { @@ -244,10 +284,10 @@ impl File { } fn get_page(&self, index: PageIndex) -> Option> { - self.pages.lock().get_page(index) + self.pages.lock().readonly_page(index) } fn get_or_allocate_page(&self, index: PageIndex) -> Arc { - self.pages.lock().get_or_allocate_page(index) + self.pages.lock().writable_page(index) } } From a61e77b35c26e8048373b6900099e45dab6169c0 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Mon, 14 Sep 2026 15:14:18 +0200 Subject: [PATCH 34/47] Re-use btree map allocation in page map on power-loss --- crates/runtime-io/src/sim/fs.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/runtime-io/src/sim/fs.rs b/crates/runtime-io/src/sim/fs.rs index ce589d64931..2bf4ec432a3 100644 --- a/crates/runtime-io/src/sim/fs.rs +++ b/crates/runtime-io/src/sim/fs.rs @@ -50,7 +50,10 @@ struct PageMap { impl PageMap { /// Reset the volatile to the durable state. fn power_loss(&mut self) { - self.volatile = self.durable.clone(); + self.volatile.clear(); + for (idx, page) in &self.durable { + self.volatile.insert(*idx, Arc::clone(page)); + } } /// Move the page at `index` from the volatile to the durable state. From d87139f1460ae8fdcd5a0b73d53deca229d90079 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 10:51:48 +0200 Subject: [PATCH 35/47] Fix task selection + use vec --- crates/runtime-io/src/sim/executor.rs | 37 +++++++++++++++++------ crates/runtime-io/src/sim/executor/sqe.rs | 14 ++++----- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 19b74963451..f5dfb040b65 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,6 +1,7 @@ use alloc::{ boxed::Box, collections::{btree_map, BTreeMap, VecDeque}, + vec::Vec, }; use core::{mem, num::NonZeroUsize, result::Result, task::Waker}; use slab::Slab; @@ -27,8 +28,8 @@ pub trait TaskSelector { fn select_tasks(&self, task_count: usize) -> impl IntoIterator; } -// TODO: There is no difference between fsync and fdatasync as long as we don't -// have an API to fsync the directory of a file after it was created. +// TODO: There is no difference between fsync and fdatasync until we extend +// [Statx] with additional fields. #[derive(Clone, Copy)] pub enum FsyncEffect { Datasync(Datasync), @@ -390,7 +391,9 @@ pub struct Executor { completions: VecDeque>, in_flight: Slab>, - executing: VecDeque, + executing: Vec, + // Scratch space for task selector + select_executing: Vec, fstree: BTreeMap, fs::File>, @@ -414,7 +417,8 @@ impl Executor { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), in_flight: Slab::with_capacity(2 * sq_capacity), - executing: VecDeque::with_capacity(2 * sq_capacity), + executing: Vec::with_capacity(2 * sq_capacity), + select_executing: Vec::with_capacity(2 * sq_capacity), fstree: BTreeMap::new(), cq_overflow, cq_dropped: 0, @@ -505,7 +509,7 @@ impl Executor { /// Drain the submission queue and advance one scheduled operation. /// - /// The operation to advance is chosen randomly using `rng`. + /// The operation to advance is chosen by `task_selector`. /// The operation is subject to `faults`. pub fn tick(&mut self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut progress = self.schedule(); @@ -551,13 +555,28 @@ impl Executor { fn execute(&mut self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut progress = false; - for index in task_selector.select_tasks(self.executing.len()) { - let op = self.executing.remove(index).expect("task index out of bounds"); + + self.select_executing.clear(); + self.select_executing.extend( + task_selector + .select_tasks(self.executing.len()) + .into_iter() + .take(self.executing.len()), + ); + self.select_executing.sort_unstable_by(|a, b| b.cmp(a)); + + let mut prev = None; + for i in 0..self.select_executing.len() { + let index = self.select_executing[i]; + assert_ne!(prev, Some(index), "duplicate task selected"); + prev = Some(index); + let op = self.executing.swap_remove(index); if let Some(delay) = self.execute_op(op, faults) { - self.executing.insert(index, delay); + self.executing.push(delay); } - progress |= true + progress |= true; } + progress } diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index f39daeef420..622fea3752b 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, collections::vec_deque::VecDeque}; +use alloc::{boxed::Box, vec::Vec}; use crate::{ sim::{ @@ -193,7 +193,7 @@ impl SqeInner { } } - pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut VecDeque) -> Pending { + pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut Vec) -> Pending { match self { SqeInner::Write { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); @@ -228,28 +228,28 @@ impl SqeInner { } } SqeInner::Open { .. } => { - executing.push_back(Executing { + executing.push(Executing { sqe: sqe_id, inner: Operation::Open, }); Pending::OneOff } SqeInner::Create { .. } => { - executing.push_back(Executing { + executing.push(Executing { sqe: sqe_id, inner: Operation::Create, }); Pending::OneOff } SqeInner::Stat { .. } => { - executing.push_back(Executing { + executing.push(Executing { sqe: sqe_id, inner: Operation::Stat, }); Pending::OneOff } SqeInner::Fallocate { .. } => { - executing.push_back(Executing { + executing.push(Executing { sqe: sqe_id, inner: Operation::Fallocate, }); @@ -298,7 +298,7 @@ impl SqeInner { } } SqeInner::Noop => { - executing.push_back(Executing { + executing.push(Executing { sqe: sqe_id, inner: Operation::Noop, }); From 32fda577edc8fa64c5a5c692eef1ba79ba02cd9c Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 11:19:30 +0200 Subject: [PATCH 36/47] Preserve `executing` allocation on `restart` --- crates/runtime-io/src/sim/executor.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index f5dfb040b65..1de9bc18a94 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -3,7 +3,7 @@ use alloc::{ collections::{btree_map, BTreeMap, VecDeque}, vec::Vec, }; -use core::{mem, num::NonZeroUsize, result::Result, task::Waker}; +use core::{num::NonZeroUsize, result::Result, task::Waker}; use slab::Slab; use crate::{ @@ -456,8 +456,7 @@ impl Executor { self.submissions.clear(); let cq_overflow_orig = self.cq_overflow; self.cq_overflow = OnCqOverflow::Drop; - let executing = mem::take(&mut self.executing); - for op in executing { + while let Some(op) = self.executing.pop() { self.execute_op(op, faults); } self.completions.clear(); From ee1c6231e83d3680656111ff76b8b30119dd5613 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 11:34:41 +0200 Subject: [PATCH 37/47] Pending capacity, options constructor, Rc instead of Arc (not Sync anyway) --- crates/runtime-io/src/sim/completion.rs | 29 ++++++++++++++---------- crates/runtime-io/src/sim/executor.rs | 28 +++++++++++++---------- crates/runtime-io/src/sim/mod.rs | 30 +++++++++++++++++-------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index 31bc11ef92b..6d9cfb135eb 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -4,7 +4,7 @@ use core::{ task::{Context, Poll, Waker}, }; -use alloc::{boxed::Box, sync::Arc}; +use alloc::{boxed::Box, rc::Rc}; use slab::Slab; use crate::{ @@ -32,12 +32,17 @@ impl CompletionState { } } -#[derive(Default)] pub(super) struct PendingCompletions { inner: Slab, } impl PendingCompletions { + pub(super) fn with_capacity(cap: usize) -> Self { + Self { + inner: Slab::with_capacity(cap), + } + } + pub(super) fn get_mut(&mut self, key: usize) -> Option<&mut CompletionHandle> { self.inner.get_mut(key) } @@ -237,13 +242,13 @@ impl CompletionHandle { } pub struct Completion { - sim: Arc, + sim: Rc, key: usize, poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll, } impl Completion, ErrorWith>>> { - pub(super) fn write(sim: Arc, key: usize) -> Self { + pub(super) fn write(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -260,7 +265,7 @@ impl Completion, ErrorWith, key: usize) -> Self { + pub(super) fn read(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -279,7 +284,7 @@ impl Completion, ErrorWith> { - pub(super) fn open(sim: Arc, key: usize) -> Self { + pub(super) fn open(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -296,7 +301,7 @@ impl Completion> { } } - pub(super) fn create(sim: Arc, key: usize) -> Self { + pub(super) fn create(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -315,7 +320,7 @@ impl Completion> { } impl Completion> { - pub(super) fn stat(sim: Arc, key: usize) -> Self { + pub(super) fn stat(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -334,7 +339,7 @@ impl Completion> { } impl Completion> { - pub(super) fn fallocate(sim: Arc, key: usize) -> Self { + pub(super) fn fallocate(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -351,7 +356,7 @@ impl Completion> { } } - pub(super) fn fsync(sim: Arc, key: usize) -> Self { + pub(super) fn fsync(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -368,7 +373,7 @@ impl Completion> { } } - pub(super) fn fdatasync(sim: Arc, key: usize) -> Self { + pub(super) fn fdatasync(sim: Rc, key: usize) -> Self { Self { sim, key, @@ -386,7 +391,7 @@ impl Completion> { } #[allow(unused)] - pub(super) fn noop(sim: Arc, key: usize) -> Self { + pub(super) fn noop(sim: Rc, key: usize) -> Self { Self { sim, key, diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 1de9bc18a94..9ce96fe3fc8 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -376,6 +376,18 @@ pub struct Options { pub cq_overflow: OnCqOverflow, } +impl Options { + pub(crate) fn sq_capacity(&self) -> usize { + self.capacity.get().next_power_of_two() + } + + pub(crate) fn cq_capacity(&self) -> usize { + self.cq_capacity + .map(|c| c.get().next_power_of_two()) + .unwrap_or_else(|| 2 * self.sq_capacity()) + } +} + impl Default for Options { fn default() -> Self { Self { @@ -402,17 +414,9 @@ pub struct Executor { } impl Executor { - pub fn new( - Options { - capacity, - cq_capacity, - cq_overflow, - }: Options, - ) -> Self { - let sq_capacity = capacity.get().next_power_of_two(); - let cq_capacity = cq_capacity - .map(|c| c.get().next_power_of_two()) - .unwrap_or_else(|| 2 * sq_capacity); + pub fn new(options: Options) -> Self { + let sq_capacity = options.sq_capacity(); + let cq_capacity = options.cq_capacity(); Self { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), @@ -420,7 +424,7 @@ impl Executor { executing: Vec::with_capacity(2 * sq_capacity), select_executing: Vec::with_capacity(2 * sq_capacity), fstree: BTreeMap::new(), - cq_overflow, + cq_overflow: options.cq_overflow, cq_dropped: 0, } } diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 1603be8037a..14a81e11923 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, sync::Arc}; +use alloc::{boxed::Box, rc::Rc, sync::Arc}; use core::result::Result; use crate::{ @@ -17,7 +17,7 @@ mod fs; pub use fs::File; pub use crate::{ - sim::executor::{FaultInjector, TaskSelector}, + sim::executor::{FaultInjector, Options, TaskSelector}, SECTOR_SIZE, }; @@ -48,10 +48,16 @@ impl From for Error { #[derive(Clone, Default)] pub struct SimulatorIO { - inner: Arc, + inner: Rc, } impl SimulatorIO { + pub fn with_options(options: Options) -> Self { + Self { + inner: Rc::new(SimulatorInner::with_options(options)), + } + } + pub fn tick(&self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut executor = self.inner.executor.lock(); @@ -103,7 +109,7 @@ impl SimulatorIO { fn submit( &self, sqe: Sqe, - completion: impl FnOnce(Arc, usize) -> Completion, + completion: impl FnOnce(Rc, usize) -> Completion, completion_handle: impl FnOnce(CompletionState>) -> CompletionHandle, ) -> Completion { let mut executor = self.inner.executor.lock(); @@ -123,7 +129,7 @@ impl SimulatorIO { fn submit_with( &self, sqe: Sqe, - completion: impl FnOnce(Arc, usize) -> Completion>, + completion: impl FnOnce(Rc, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, ) -> Completion> { let mut executor = self.inner.executor.lock(); @@ -154,15 +160,21 @@ struct SimulatorInner { pending: spin::Mutex, } -impl Default for SimulatorInner { - fn default() -> Self { +impl SimulatorInner { + fn with_options(options: Options) -> Self { Self { - executor: spin::Mutex::new(Executor::new(<_>::default())), - pending: <_>::default(), + pending: spin::Mutex::new(PendingCompletions::with_capacity(options.cq_capacity())), + executor: spin::Mutex::new(Executor::new(options)), } } } +impl Default for SimulatorInner { + fn default() -> Self { + Self::with_options(<_>::default()) + } +} + impl SpacetimeIO for SimulatorIO { type Fd = fs::File; type Error = Error; From 7b7da4e5b7fb89c4bc6d9851cc674436d9bed307 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 13:21:16 +0200 Subject: [PATCH 38/47] Bound executing queue --- crates/runtime-io/src/sim/executor.rs | 74 ++++++----- crates/runtime-io/src/sim/executor/sqe.rs | 143 ++++++++++++---------- crates/runtime-io/src/sim/mod.rs | 2 +- 3 files changed, 121 insertions(+), 98 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 9ce96fe3fc8..dfd9564cc5f 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -369,11 +369,17 @@ pub struct Options { /// queue's. This can be insufficient for some workloads, so this setting /// can be used to override the default. /// - /// Should be a power of 2, or is otherwise rounder up to the next power of + /// Should be a power of 2, or is otherwise rounded up to the next power of /// two. pub cq_capacity: Option, /// What to do if the completion queue overflows. pub cq_overflow: OnCqOverflow, + /// Bound on the number of concurrently executing tasks. + /// + /// Note that one [Sqe] can result in many operations to be scheduled. + /// Should be a power of 2, or is otherwise rounded up to the next power of + /// two. + pub max_concurrency: NonZeroUsize, } impl Options { @@ -386,6 +392,10 @@ impl Options { .map(|c| c.get().next_power_of_two()) .unwrap_or_else(|| 2 * self.sq_capacity()) } + + pub(crate) fn max_concurrency(&self) -> usize { + self.max_concurrency.get().next_power_of_two() + } } impl Default for Options { @@ -394,6 +404,7 @@ impl Default for Options { capacity: NonZeroUsize::new(8).unwrap(), cq_capacity: None, cq_overflow: OnCqOverflow::default(), + max_concurrency: NonZeroUsize::new(32).unwrap(), } } } @@ -421,8 +432,8 @@ impl Executor { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), in_flight: Slab::with_capacity(2 * sq_capacity), - executing: Vec::with_capacity(2 * sq_capacity), - select_executing: Vec::with_capacity(2 * sq_capacity), + executing: Vec::with_capacity(options.max_concurrency()), + select_executing: Vec::with_capacity(options.max_concurrency()), fstree: BTreeMap::new(), cq_overflow: options.cq_overflow, cq_dropped: 0, @@ -524,31 +535,32 @@ impl Executor { let mut progress = false; while let Some(mut sqe) = self.submissions.pop_front() { - // If the sqe is linked, pop the whole chain. - // Links of sqes not submitted in the same batch are ignored. - let mut successors = VecDeque::new(); - if let Some(link) = sqe.link { - let mut link_kind = link; - while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { - successors.push_back(Blocked { - link: link_kind, - sqe: inner, - user_data, - }); - match link { - Some(kind) => link_kind = kind, - None => break, + let slot = self.in_flight.vacant_entry(); + if let Some(pending) = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing) { + // If the sqe is linked, pop the whole chain. + // Links of sqes not submitted in the same batch are ignored. + let mut successors = VecDeque::new(); + if let Some(link) = sqe.link { + let mut link_kind = link; + while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { + successors.push_back(Blocked { + link: link_kind, + sqe: inner, + user_data, + }); + match link { + Some(kind) => link_kind = kind, + None => break, + } } } + slot.insert(InFlight { + sqe: sqe.inner, + pending, + blocked: successors, + user_data: sqe.user_data, + }); } - let slot = self.in_flight.vacant_entry(); - let pending = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing); - slot.insert(InFlight { - sqe: sqe.inner, - pending, - blocked: successors, - user_data: sqe.user_data, - }); progress = true } @@ -748,11 +760,13 @@ impl Executor { else { unreachable!("invalid sqe: expected create") }; - let run = |()| match self.fstree.entry(path) { - btree_map::Entry::Vacant(entry) => Ok(entry.insert(fs::File::default()).clone()), - btree_map::Entry::Occupied(entry) => Err(Error::FileAlreadyExists { - path: entry.key().clone(), - }), + let run = |()| { + // Avoid cloning `path` if already exists. + if self.fstree.contains_key(&path) { + Err(Error::FileAlreadyExists { path }) + } else { + Ok(self.fstree.entry(path).or_insert_with(fs::File::default).clone()) + } }; let result = eff.traverse(run, Err); let is_success = result.is_ok(); diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 622fea3752b..608458602aa 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -193,117 +193,126 @@ impl SqeInner { } } - pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut Vec) -> Pending { + pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut Vec) -> Option { + let exe_cap = executing.spare_capacity_mut().len(); match self { SqeInner::Write { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - executing.extend((0..page_count).map(|page| Executing { - sqe: sqe_id, - inner: Operation::WriteSector(WriteSector { - page_offset: first_sector + page, - buf_offset: page * SECTOR_SIZE, - }), - })); - Pending::ReadWrite { - results: Results::new(page_count), - } + (exe_cap >= page_count).then(|| { + executing.extend((0..page_count).map(|page| Executing { + sqe: sqe_id, + inner: Operation::WriteSector(WriteSector { + page_offset: first_sector + page, + buf_offset: page * SECTOR_SIZE, + }), + })); + Pending::ReadWrite { + results: Results::new(page_count), + } + }) } SqeInner::Read { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - executing.extend((0..page_count).map(|page| Executing { - sqe: sqe_id, - inner: Operation::ReadSector(ReadSector { - page_offset: first_sector + page, - buf_offset: page * SECTOR_SIZE, - }), - })); - Pending::ReadWrite { - results: Results::new(page_count), - } + (exe_cap >= page_count).then(|| { + executing.extend((0..page_count).map(|page| Executing { + sqe: sqe_id, + inner: Operation::ReadSector(ReadSector { + page_offset: first_sector + page, + buf_offset: page * SECTOR_SIZE, + }), + })); + Pending::ReadWrite { + results: Results::new(page_count), + } + }) } - SqeInner::Open { .. } => { + SqeInner::Open { .. } => (exe_cap >= 1).then(|| { executing.push(Executing { sqe: sqe_id, inner: Operation::Open, }); Pending::OneOff - } - SqeInner::Create { .. } => { + }), + SqeInner::Create { .. } => (exe_cap >= 1).then(|| { executing.push(Executing { sqe: sqe_id, inner: Operation::Create, }); Pending::OneOff - } - SqeInner::Stat { .. } => { + }), + SqeInner::Stat { .. } => (exe_cap >= 1).then(|| { executing.push(Executing { sqe: sqe_id, inner: Operation::Stat, }); Pending::OneOff - } - SqeInner::Fallocate { .. } => { + }), + SqeInner::Fallocate { .. } => (exe_cap >= 1).then(|| { executing.push(Executing { sqe: sqe_id, inner: Operation::Fallocate, }); Pending::OneOff - } + }), SqeInner::Fsync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; - executing.extend( - (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Sector(offset)), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Length), - }, - }]), - ); - Pending::Sync { - results: Results::new(1 + sector_count as usize), - } + (exe_cap > sector_count as usize).then(|| { + executing.extend( + (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Sector(offset)), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Length), + }, + }]), + ); + Pending::Sync { + results: Results::new(1 + sector_count as usize), + } + }) } SqeInner::Fdatasync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; - executing.extend( - (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Sector(offset), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Length, - }, - }]), - ); - Pending::Sync { - results: Results::new(1 + sector_count as usize), - } + (exe_cap > sector_count as usize).then(|| { + executing.extend( + (0..sector_count) + .map(|offset| Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Sector(offset), + }, + }) + .chain([Executing { + sqe: sqe_id, + inner: Operation::Fdatasync { + effect: Datasync::Length, + }, + }]), + ); + Pending::Sync { + results: Results::new(1 + sector_count as usize), + } + }) } - SqeInner::Noop => { + SqeInner::Noop => (exe_cap >= 1).then(|| { executing.push(Executing { sqe: sqe_id, inner: Operation::Noop, }); Pending::OneOff - } + }), } } } diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 14a81e11923..80c7d0ac301 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, rc::Rc, sync::Arc}; +use alloc::{boxed::Box, rc::Rc}; use core::result::Result; use crate::{ From f0e302e524ac26a6fe93e2bebf1091cd865989cc Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 13:22:39 +0200 Subject: [PATCH 39/47] Avoid alloc when trimming file --- crates/runtime-io/src/sim/fs.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/runtime-io/src/sim/fs.rs b/crates/runtime-io/src/sim/fs.rs index 2bf4ec432a3..3c73f0525cc 100644 --- a/crates/runtime-io/src/sim/fs.rs +++ b/crates/runtime-io/src/sim/fs.rs @@ -114,8 +114,7 @@ impl PageMap { } Less => { let first_removed = PageIndex::from_offset(new_len); - let removed = page_map.split_off(&first_removed); - drop(removed); + page_map.retain(|&index, _| index < first_removed); } } } From 108ac4c9cf8030c149b6af06ca6feccc920c5f75 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Tue, 15 Sep 2026 14:30:02 +0200 Subject: [PATCH 40/47] Track blocked linked in-flight tasks in the same queue, and make that queue bounded. --- crates/runtime-io/src/sim/executor.rs | 271 ++++++++++++++-------- crates/runtime-io/src/sim/executor/sqe.rs | 1 - crates/runtime-io/src/sim/mod.rs | 2 +- 3 files changed, 176 insertions(+), 98 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index dfd9564cc5f..e12c1f0d035 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,6 +1,6 @@ use alloc::{ boxed::Box, - collections::{btree_map, BTreeMap, VecDeque}, + collections::{BTreeMap, VecDeque}, vec::Vec, }; use core::{num::NonZeroUsize, result::Result, task::Waker}; @@ -150,17 +150,50 @@ impl CqeInner { } } -pub struct Blocked { - pub link: LinkKind, +pub struct InFlight { pub sqe: SqeInner, + pub state: InFlightState, + next: Option, pub user_data: Option, } -pub struct InFlight { - pub sqe: SqeInner, - pub pending: Pending, - pub blocked: VecDeque>, - pub user_data: Option, +impl InFlight { + fn is_blocked(&self) -> bool { + self.state.is_blocked() + } + + fn is_ready(&self) -> bool { + self.state.is_ready() + } + + fn is_active(&self) -> bool { + self.state.is_active() + } +} + +pub enum InFlightState { + Blocked, + Ready, + Active(Pending), +} + +impl InFlightState { + fn is_blocked(&self) -> bool { + matches!(self, Self::Blocked) + } + + fn is_ready(&self) -> bool { + matches!(self, Self::Ready) + } + + fn is_active(&self) -> bool { + matches!(self, Self::Active(_)) + } +} + +struct Link { + kind: LinkKind, + next: SqeId, } pub trait ResultAcc { @@ -531,38 +564,83 @@ impl Executor { progress } + fn schedule_blocked(&mut self, sqe: SqeInner, user_data: Option) -> SqeId { + let entry = self.in_flight.vacant_entry(); + let sqe_id = SqeId(entry.key()); + + entry.insert(InFlight { + sqe, + state: InFlightState::Blocked, + next: None, + user_data, + }); + + sqe_id + } + + fn unblock(&mut self, sqe: SqeId) { + let Self { in_flight, .. } = self; + + let in_flight = in_flight.get_mut(sqe.key()).expect("invalid sqe id"); + assert!(in_flight.is_blocked(), "in-flight sqe unblocked more than once"); + in_flight.state = InFlightState::Ready; + } + + fn have_in_flight_capacity(&self) -> bool { + // The batch size if the prefix of linked SQEs, plus the first unlinked + // one. They all need to be scheduled together to presever linking + // semantics. + let batch_size = self + .submissions + .iter() + .position(|sqe| !sqe.is_linked()) + .map_or(self.submissions.len(), |i| i + 1); + + batch_size <= self.in_flight.capacity() - self.in_flight.len() + } + fn schedule(&mut self) -> bool { let mut progress = false; - while let Some(mut sqe) = self.submissions.pop_front() { - let slot = self.in_flight.vacant_entry(); - if let Some(pending) = sqe.inner.schedule(SqeId(slot.key()), &mut self.executing) { - // If the sqe is linked, pop the whole chain. - // Links of sqes not submitted in the same batch are ignored. - let mut successors = VecDeque::new(); - if let Some(link) = sqe.link { - let mut link_kind = link; - while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { - successors.push_back(Blocked { - link: link_kind, - sqe: inner, - user_data, - }); - match link { - Some(kind) => link_kind = kind, - None => break, - } - } - } - slot.insert(InFlight { - sqe: sqe.inner, - pending, - blocked: successors, - user_data: sqe.user_data, - }); + if !self.have_in_flight_capacity() { + return true; + } + + while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { + let head = self.schedule_blocked(inner, user_data); + + let mut prev = head; + let mut prev_link = link; + while let Some(kind) = prev_link { + let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() else { + break; + }; + + let next = self.schedule_blocked(inner, user_data); + let prev_in_flight = self.in_flight.get_mut(prev.key()).expect("invalid sqe id"); + prev_in_flight.next = Some(Link { kind, next }); + + prev = next; + prev_link = link; + } + + self.unblock(head); + progress |= true; + + if !self.have_in_flight_capacity() { + break; + } + } + + for (key, in_flight) in self.in_flight.iter_mut() { + if in_flight.is_ready() { + let Some(pending) = in_flight.sqe.schedule(SqeId(key), &mut self.executing) else { + break; + }; + in_flight.state = InFlightState::Active(pending); } - progress = true + progress |= true; } progress @@ -598,6 +676,8 @@ impl Executor { fn execute_op(&mut self, op: Executing, faults: &mut impl FaultInjector) -> Option { op.traverse(|sqe, op| { let in_flight = self.in_flight.get(sqe.key()).expect("invalid sqe id"); + assert!(in_flight.is_active()); + match op { Operation::WriteSector(effect) => faults .inject_write_sector_fault(in_flight, effect) @@ -643,7 +723,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Write { fd, buf, .. }, - pending: Pending::ReadWrite { results }, + state: InFlightState::Active(Pending::ReadWrite { results }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -666,8 +746,8 @@ impl Executor { if is_complete { let InFlight { sqe: SqeInner::Write { buf, .. }, - pending: Pending::ReadWrite { results }, - blocked, + state: InFlightState::Active(Pending::ReadWrite { results }), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -679,7 +759,7 @@ impl Executor { inner: CqeInner::Write { result, buf }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } } @@ -687,7 +767,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Read { fd, buf, .. }, - pending: Pending::ReadWrite { results }, + state: InFlightState::Active(Pending::ReadWrite { results }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -710,8 +790,8 @@ impl Executor { if is_complete { let InFlight { sqe: SqeInner::Read { buf, .. }, - pending: Pending::ReadWrite { results }, - blocked, + state: InFlightState::Active(Pending::ReadWrite { results }), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -723,15 +803,15 @@ impl Executor { inner: CqeInner::Read { result, buf }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } } fn execute_open(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Open { path }, - pending: Pending::OneOff, - blocked, + state: InFlightState::Active(Pending::OneOff), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -747,14 +827,14 @@ impl Executor { inner: CqeInner::Open { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } fn execute_create(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Create { path }, - pending: Pending::OneOff, - blocked, + state: InFlightState::Active(Pending::OneOff), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -765,7 +845,7 @@ impl Executor { if self.fstree.contains_key(&path) { Err(Error::FileAlreadyExists { path }) } else { - Ok(self.fstree.entry(path).or_insert_with(fs::File::default).clone()) + Ok(self.fstree.entry(path).or_default().clone()) } }; let result = eff.traverse(run, Err); @@ -774,14 +854,14 @@ impl Executor { inner: CqeInner::Create { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } fn execute_stat(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Stat { fd }, - pending: Pending::OneOff, - blocked, + state: InFlightState::Active(Pending::OneOff), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -793,14 +873,14 @@ impl Executor { inner: CqeInner::Stat { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } fn execute_fallocate(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Fallocate { fd, total_len }, - pending: Pending::OneOff, - blocked, + state: InFlightState::Active(Pending::OneOff), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -812,14 +892,14 @@ impl Executor { inner: CqeInner::Fallocate { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } fn execute_fsync(&mut self, sqe: SqeId, eff: EitherOrBoth) { let is_complete = { let InFlight { sqe: SqeInner::Fsync { fd }, - pending: Pending::Sync { results }, + state: InFlightState::Active(Pending::Sync { results }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -838,8 +918,8 @@ impl Executor { if is_complete { let InFlight { - pending: Pending::Sync { results }, - blocked, + state: InFlightState::Active(Pending::Sync { results }), + next, user_data, .. } = self.in_flight.remove(sqe.key()) @@ -852,7 +932,7 @@ impl Executor { inner: CqeInner::Fsync { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } } @@ -860,7 +940,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Fdatasync { fd }, - pending: Pending::Sync { results }, + state: InFlightState::Active(Pending::Sync { results }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -879,8 +959,8 @@ impl Executor { if is_complete { let InFlight { - pending: Pending::Sync { results }, - blocked, + state: InFlightState::Active(Pending::Sync { results }), + next, user_data, .. } = self.in_flight.remove(sqe.key()) @@ -893,15 +973,15 @@ impl Executor { inner: CqeInner::Fdatasync { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } } fn execute_noop(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Noop, - pending: Pending::OneOff, - blocked, + state: InFlightState::Active(Pending::OneOff), + next, user_data, } = self.in_flight.remove(sqe.key()) else { @@ -913,39 +993,38 @@ impl Executor { inner: CqeInner::Noop { result }, user_data, }); - self.schedule_linked(sqe, is_success, blocked); + self.schedule_linked(is_success, next); } - fn schedule_linked(&mut self, sqe: SqeId, prev_succeeded: bool, mut blocked: VecDeque>) { - if let Some(Blocked { - link, - sqe: mut next, - user_data, - }) = blocked.pop_front() - { - match (link, prev_succeeded) { - (LinkKind::Soft, false) => { - self.complete(next.cancel(user_data)); - for Blocked { - link: _, - sqe: next, - user_data, - } in blocked - { - self.complete(next.cancel(user_data)); - } - } - (LinkKind::Soft, true) | (LinkKind::Hard, _) => { - let pending = next.schedule(sqe, &mut self.executing); - let slot = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id"); - *slot = InFlight { - sqe: next, - pending, - blocked, - user_data, - }; - } + fn schedule_linked(&mut self, prev_succeeded: bool, next: Option) { + let Some(Link { kind, next }) = next else { + return; + }; + match (kind, prev_succeeded) { + (LinkKind::Hard, _) | (LinkKind::Soft, true) => { + self.unblock(next); + } + (LinkKind::Soft, false) => { + self.cancel_chain(next); } } } + + fn cancel_chain(&mut self, mut head: SqeId) { + loop { + let InFlight { + sqe, + state, + next, + user_data, + } = self.in_flight.remove(head.key()); + assert!(state.is_blocked(), "attempted to cancel an active linked sqe"); + + self.complete(sqe.cancel(user_data)); + let Some(Link { next, .. }) = next else { + break; + }; + head = next; + } + } } diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 608458602aa..bcbe7810acc 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -47,7 +47,6 @@ impl Sqe { self } - #[allow(unused)] pub fn is_linked(&self) -> bool { self.link.is_some() } diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 80c7d0ac301..f852ae5d809 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -17,7 +17,7 @@ mod fs; pub use fs::File; pub use crate::{ - sim::executor::{FaultInjector, Options, TaskSelector}, + sim::executor::{FaultInjector, LinkKind, Options, TaskSelector}, SECTOR_SIZE, }; From 036f5544faa07c848d809bd0dd7d765d799b710b Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 16 Sep 2026 10:47:24 +0200 Subject: [PATCH 41/47] Feed `executing` incrementally --- crates/runtime-io/src/sim/executor.rs | 154 ++++++++++------- crates/runtime-io/src/sim/executor/sqe.rs | 199 +++++++++------------- 2 files changed, 172 insertions(+), 181 deletions(-) diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index e12c1f0d035..c8f942b648f 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -3,7 +3,13 @@ use alloc::{ collections::{BTreeMap, VecDeque}, vec::Vec, }; -use core::{num::NonZeroUsize, result::Result, task::Waker}; +use core::{ + iter::{Chain, Map, Scan}, + num::NonZeroUsize, + ops::Range, + result::Result, + task::Waker, +}; use slab::Slab; use crate::{ @@ -35,6 +41,12 @@ pub enum FsyncEffect { Datasync(Datasync), } +impl From for FsyncEffect { + fn from(value: Datasync) -> Self { + Self::Datasync(value) + } +} + #[derive(Clone, Copy)] pub enum Operation { WriteSector(WriteSector), @@ -151,8 +163,8 @@ impl CqeInner { } pub struct InFlight { - pub sqe: SqeInner, - pub state: InFlightState, + sqe: SqeInner, + state: InFlightState, next: Option, pub user_data: Option, } @@ -162,18 +174,13 @@ impl InFlight { self.state.is_blocked() } - fn is_ready(&self) -> bool { - self.state.is_ready() - } - fn is_active(&self) -> bool { self.state.is_active() } } -pub enum InFlightState { +enum InFlightState { Blocked, - Ready, Active(Pending), } @@ -182,10 +189,6 @@ impl InFlightState { matches!(self, Self::Blocked) } - fn is_ready(&self) -> bool { - matches!(self, Self::Ready) - } - fn is_active(&self) -> bool { matches!(self, Self::Active(_)) } @@ -251,10 +254,25 @@ impl Results { } } -pub enum Pending { - OneOff, - ReadWrite { results: Results }, - Sync { results: Results<()> }, +type ReadWriteOps = Scan, usize, fn(&mut usize, usize) -> Option>; +type SyncOps = Chain, fn(u64) -> Operation>, core::option::IntoIter>; + +enum Pending { + ReadWrite { ops: ReadWriteOps, results: Results }, + Sync { ops: SyncOps, results: Results<()> }, + Unit { op: Option }, +} + +impl Iterator for Pending { + type Item = Operation; + + fn next(&mut self) -> Option { + match self { + Self::ReadWrite { ops, .. } => ops.next(), + Self::Sync { ops, .. } => ops.next(), + Self::Unit { op } => op.take(), + } + } } struct Executing { @@ -505,8 +523,14 @@ impl Executor { let cq_overflow_orig = self.cq_overflow; self.cq_overflow = OnCqOverflow::Drop; while let Some(op) = self.executing.pop() { + if let Some(in_flight) = self.in_flight.get(op.sqe.key()) + && !in_flight.is_active() + { + continue; + } self.execute_op(op, faults); } + self.in_flight.clear(); self.completions.clear(); self.cq_overflow = cq_overflow_orig; self.cq_dropped = 0; @@ -519,7 +543,7 @@ impl Executor { Batch::IntoIter: ExactSizeIterator, { let sqes = sqes.into_iter(); - if self.submissions.len() + sqes.len() >= self.submissions.capacity() { + if self.submissions.len() + sqes.len() > self.submissions.capacity() { Err(sqes) } else { self.submissions.extend(sqes); @@ -583,12 +607,12 @@ impl Executor { let in_flight = in_flight.get_mut(sqe.key()).expect("invalid sqe id"); assert!(in_flight.is_blocked(), "in-flight sqe unblocked more than once"); - in_flight.state = InFlightState::Ready; + in_flight.state = InFlightState::Active(in_flight.sqe.prepare()); } fn have_in_flight_capacity(&self) -> bool { - // The batch size if the prefix of linked SQEs, plus the first unlinked - // one. They all need to be scheduled together to presever linking + // The batch size is the prefix of linked SQEs, plus the first unlinked + // one. They all need to be scheduled together to preserve linking // semantics. let batch_size = self .submissions @@ -602,45 +626,49 @@ impl Executor { fn schedule(&mut self) -> bool { let mut progress = false; - if !self.have_in_flight_capacity() { - return true; - } - - while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { - let head = self.schedule_blocked(inner, user_data); + if self.have_in_flight_capacity() { + while let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() { + let head = self.schedule_blocked(inner, user_data); - let mut prev = head; - let mut prev_link = link; - while let Some(kind) = prev_link { - let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() else { - break; - }; + let mut prev = head; + let mut prev_link = link; + while let Some(kind) = prev_link { + let Some(Sqe { inner, link, user_data }) = self.submissions.pop_front() else { + break; + }; - let next = self.schedule_blocked(inner, user_data); - let prev_in_flight = self.in_flight.get_mut(prev.key()).expect("invalid sqe id"); - prev_in_flight.next = Some(Link { kind, next }); + let next = self.schedule_blocked(inner, user_data); + let prev_in_flight = self.in_flight.get_mut(prev.key()).expect("invalid sqe id"); + prev_in_flight.next = Some(Link { kind, next }); - prev = next; - prev_link = link; - } + prev = next; + prev_link = link; + } - self.unblock(head); - progress |= true; + self.unblock(head); + progress |= true; - if !self.have_in_flight_capacity() { - break; + if !self.have_in_flight_capacity() { + break; + } } } - for (key, in_flight) in self.in_flight.iter_mut() { - if in_flight.is_ready() { - let Some(pending) = in_flight.sqe.schedule(SqeId(key), &mut self.executing) else { - break; - }; - in_flight.state = InFlightState::Active(pending); + for (sqe_id, in_flight) in self.in_flight.iter_mut() { + if self.executing.capacity() == self.executing.len() { + break; } - progress |= true; + let InFlightState::Active(pending) = &mut in_flight.state else { + continue; + }; + if let Some(op) = pending.next() { + self.executing.push(Executing { + sqe: SqeId(sqe_id), + inner: op, + }); + progress |= true; + } } progress @@ -723,7 +751,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Write { fd, buf, .. }, - state: InFlightState::Active(Pending::ReadWrite { results }), + state: InFlightState::Active(Pending::ReadWrite { results, .. }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -746,7 +774,7 @@ impl Executor { if is_complete { let InFlight { sqe: SqeInner::Write { buf, .. }, - state: InFlightState::Active(Pending::ReadWrite { results }), + state: InFlightState::Active(Pending::ReadWrite { results, .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -767,7 +795,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Read { fd, buf, .. }, - state: InFlightState::Active(Pending::ReadWrite { results }), + state: InFlightState::Active(Pending::ReadWrite { results, .. }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -790,7 +818,7 @@ impl Executor { if is_complete { let InFlight { sqe: SqeInner::Read { buf, .. }, - state: InFlightState::Active(Pending::ReadWrite { results }), + state: InFlightState::Active(Pending::ReadWrite { results, .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -810,7 +838,7 @@ impl Executor { fn execute_open(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Open { path }, - state: InFlightState::Active(Pending::OneOff), + state: InFlightState::Active(Pending::Unit { .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -833,7 +861,7 @@ impl Executor { fn execute_create(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Create { path }, - state: InFlightState::Active(Pending::OneOff), + state: InFlightState::Active(Pending::Unit { .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -860,7 +888,7 @@ impl Executor { fn execute_stat(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Stat { fd }, - state: InFlightState::Active(Pending::OneOff), + state: InFlightState::Active(Pending::Unit { .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -879,7 +907,7 @@ impl Executor { fn execute_fallocate(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Fallocate { fd, total_len }, - state: InFlightState::Active(Pending::OneOff), + state: InFlightState::Active(Pending::Unit { .. }), next, user_data, } = self.in_flight.remove(sqe.key()) @@ -899,7 +927,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Fsync { fd }, - state: InFlightState::Active(Pending::Sync { results }), + state: InFlightState::Active(Pending::Sync { results, .. }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -918,7 +946,7 @@ impl Executor { if is_complete { let InFlight { - state: InFlightState::Active(Pending::Sync { results }), + state: InFlightState::Active(Pending::Sync { results, .. }), next, user_data, .. @@ -940,7 +968,7 @@ impl Executor { let is_complete = { let InFlight { sqe: SqeInner::Fdatasync { fd }, - state: InFlightState::Active(Pending::Sync { results }), + state: InFlightState::Active(Pending::Sync { results, .. }), .. } = self.in_flight.get_mut(sqe.key()).expect("invalid sqe id") else { @@ -959,7 +987,7 @@ impl Executor { if is_complete { let InFlight { - state: InFlightState::Active(Pending::Sync { results }), + state: InFlightState::Active(Pending::Sync { results, .. }), next, user_data, .. @@ -980,7 +1008,7 @@ impl Executor { fn execute_noop(&mut self, sqe: SqeId, eff: EitherOrBoth<(), Error>) { let InFlight { sqe: SqeInner::Noop, - state: InFlightState::Active(Pending::OneOff), + state: InFlightState::Active(Pending::Unit { .. }), next, user_data, } = self.in_flight.remove(sqe.key()) diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index bcbe7810acc..8388b50db58 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -1,8 +1,8 @@ -use alloc::{boxed::Box, vec::Vec}; +use alloc::boxed::Box; use crate::{ sim::{ - executor::{Cqe, CqeInner, Executing, FsyncEffect, Operation, Pending, ReadSector, Results, WriteSector}, + executor::{Cqe, CqeInner, FsyncEffect, Operation, Pending, ReadSector, Results, WriteSector}, fs::{self, Datasync}, Error, }, @@ -41,6 +41,14 @@ pub struct Sqe { } impl Sqe { + fn new(inner: SqeInner) -> Self { + Self { + inner, + link: None, + user_data: None, + } + } + #[allow(unused)] pub fn link(mut self, kind: Option) -> Self { self.link = kind; @@ -57,40 +65,42 @@ impl Sqe { } pub fn write(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - SqeInner::Write { fd, buf, offset }.into() + assert!(offset.is_multiple_of(SECTOR_SIZE as u64)); + Self::new(SqeInner::Write { fd, buf, offset }) } pub fn read(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - SqeInner::Read { fd, buf, offset }.into() + assert!(offset.is_multiple_of(SECTOR_SIZE as u64)); + Self::new(SqeInner::Read { fd, buf, offset }) } pub fn open(path: Box) -> Self { - SqeInner::Open { path }.into() + Self::new(SqeInner::Open { path }) } pub fn create(path: Box) -> Self { - SqeInner::Create { path }.into() + Self::new(SqeInner::Create { path }) } pub fn stat(fd: fs::File) -> Self { - SqeInner::Stat { fd }.into() + Self::new(SqeInner::Stat { fd }) } pub fn fallocate(fd: fs::File, len: u64) -> Self { - SqeInner::Fallocate { fd, total_len: len }.into() + Self::new(SqeInner::Fallocate { fd, total_len: len }) } pub fn fsync(fd: fs::File) -> Self { - SqeInner::Fsync { fd }.into() + Self::new(SqeInner::Fsync { fd }) } pub fn fdatasync(fd: fs::File) -> Self { - SqeInner::Fdatasync { fd }.into() + Self::new(SqeInner::Fdatasync { fd }) } #[allow(unused)] pub fn noop() -> Self { - SqeInner::Noop.into() + Self::new(SqeInner::Noop) } /// Extract the [ErasedBox] buffer if the [Sqe] carries one. @@ -108,16 +118,6 @@ impl Sqe { } } -impl> From for Sqe { - fn from(inner: U) -> Self { - Self { - inner: inner.into(), - link: None, - user_data: None, - } - } -} - pub enum SqeInner { Write { fd: fs::File, buf: ErasedBox, offset: u64 }, Read { fd: fs::File, buf: ErasedBox, offset: u64 }, @@ -192,126 +192,89 @@ impl SqeInner { } } - pub(super) fn schedule(&mut self, sqe_id: SqeId, executing: &mut Vec) -> Option { - let exe_cap = executing.spare_capacity_mut().len(); + pub(super) fn prepare(&self) -> Pending { match self { SqeInner::Write { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - (exe_cap >= page_count).then(|| { - executing.extend((0..page_count).map(|page| Executing { - sqe: sqe_id, - inner: Operation::WriteSector(WriteSector { - page_offset: first_sector + page, + Pending::ReadWrite { + ops: (0..page_count).scan(first_sector, |first_sector, page| { + Some(Operation::WriteSector(WriteSector { + page_offset: *first_sector + page, buf_offset: page * SECTOR_SIZE, - }), - })); - Pending::ReadWrite { - results: Results::new(page_count), - } - }) + })) + }), + results: Results::new(page_count), + } } SqeInner::Read { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); let first_sector = (*offset / SECTOR_SIZE as u64) as usize; let page_count = buf_len / SECTOR_SIZE; - (exe_cap >= page_count).then(|| { - executing.extend((0..page_count).map(|page| Executing { - sqe: sqe_id, - inner: Operation::ReadSector(ReadSector { - page_offset: first_sector + page, + Pending::ReadWrite { + ops: (0..page_count).scan(first_sector, |first_sector, page| { + Some(Operation::ReadSector(ReadSector { + page_offset: *first_sector + page, buf_offset: page * SECTOR_SIZE, - }), - })); - Pending::ReadWrite { - results: Results::new(page_count), - } - }) + })) + }), + results: Results::new(page_count), + } } - SqeInner::Open { .. } => (exe_cap >= 1).then(|| { - executing.push(Executing { - sqe: sqe_id, - inner: Operation::Open, - }); - Pending::OneOff - }), - SqeInner::Create { .. } => (exe_cap >= 1).then(|| { - executing.push(Executing { - sqe: sqe_id, - inner: Operation::Create, - }); - Pending::OneOff - }), - SqeInner::Stat { .. } => (exe_cap >= 1).then(|| { - executing.push(Executing { - sqe: sqe_id, - inner: Operation::Stat, - }); - Pending::OneOff - }), - SqeInner::Fallocate { .. } => (exe_cap >= 1).then(|| { - executing.push(Executing { - sqe: sqe_id, - inner: Operation::Fallocate, - }); - Pending::OneOff - }), SqeInner::Fsync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; - (exe_cap > sector_count as usize).then(|| { - executing.extend( - (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Sector(offset)), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Length), - }, - }]), - ); - Pending::Sync { - results: Results::new(1 + sector_count as usize), + + let f = |offset: u64| -> Operation { + Operation::Fsync { + effect: FsyncEffect::Datasync(Datasync::Sector(offset)), } - }) + }; + + Pending::Sync { + ops: (0..sector_count) + .map(f as fn(u64) -> Operation) + .chain(Some(Operation::Fsync { + effect: Datasync::Length.into(), + })), + results: Results::new(1 + sector_count as usize), + } } SqeInner::Fdatasync { fd } => { let sector_count = fd.len() / SECTOR_SIZE as u64; - (exe_cap > sector_count as usize).then(|| { - executing.extend( - (0..sector_count) - .map(|offset| Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Sector(offset), - }, - }) - .chain([Executing { - sqe: sqe_id, - inner: Operation::Fdatasync { - effect: Datasync::Length, - }, - }]), - ); - Pending::Sync { - results: Results::new(1 + sector_count as usize), + + let f = |offset: u64| -> Operation { + Operation::Fdatasync { + effect: Datasync::Sector(offset), } - }) + }; + + Pending::Sync { + ops: (0..sector_count) + .map(f as fn(u64) -> Operation) + .chain(Some(Operation::Fdatasync { + effect: Datasync::Length, + })), + results: Results::new(1 + sector_count as usize), + } } - SqeInner::Noop => (exe_cap >= 1).then(|| { - executing.push(Executing { - sqe: sqe_id, - inner: Operation::Noop, - }); - Pending::OneOff - }), + SqeInner::Open { .. } => Pending::Unit { + op: Some(Operation::Open), + }, + SqeInner::Create { .. } => Pending::Unit { + op: Some(Operation::Create), + }, + SqeInner::Stat { .. } => Pending::Unit { + op: Some(Operation::Stat), + }, + SqeInner::Fallocate { .. } => Pending::Unit { + op: Some(Operation::Fallocate), + }, + SqeInner::Noop => Pending::Unit { + op: Some(Operation::Noop), + }, } } } From 685a48586d9fecc78a9ff1180902c1cf8cd01fb8 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 16 Sep 2026 15:03:57 +0200 Subject: [PATCH 42/47] Make pending completions bounded --- crates/runtime-io/src/sim/completion.rs | 154 +++++++++++++++--------- crates/runtime-io/src/sim/mod.rs | 80 +++++++----- 2 files changed, 146 insertions(+), 88 deletions(-) diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index 6d9cfb135eb..4b3f62a80ae 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -8,7 +8,7 @@ use alloc::{boxed::Box, rc::Rc}; use slab::Slab; use crate::{ - sim::{fs, Error, SimulatorInner}, + sim::{fs, Error}, AlignedBytes, ErasedBox, ErrorWith, Statx, }; @@ -51,8 +51,12 @@ impl PendingCompletions { self.inner.clear(); } - pub(super) fn vacant_entry(&mut self) -> VacantEntry<'_, CompletionHandle> { - self.inner.vacant_entry() + pub(super) fn vacant_entry(&mut self) -> Option> { + if self.inner.capacity() == self.inner.len() { + None + } else { + Some(self.inner.vacant_entry()) + } } fn remove(&mut self, key: usize) -> CompletionHandle { @@ -242,19 +246,23 @@ impl CompletionHandle { } pub struct Completion { - sim: Rc, - key: usize, - poll: fn(&SimulatorInner, usize, &mut Context<'_>) -> Poll, + inner: CompletionInner, +} + +impl Completion { + pub(super) fn ready(val: T) -> Self { + CompletionInner::Ready(Some(val)).into() + } } impl Completion, ErrorWith>>> { - pub(super) fn write(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn write(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::write_state_mut, CompletionHandle::into_write_state, @@ -263,15 +271,16 @@ impl Completion, ErrorWith, key: usize) -> Self { - Self { - sim, + pub(super) fn read(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::read_state_mut, CompletionHandle::into_read_state, @@ -280,17 +289,18 @@ impl Completion, ErrorWith> { - pub(super) fn open(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn open(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::open_state_mut, CompletionHandle::into_open_state, @@ -299,15 +309,16 @@ impl Completion> { ) }, } + .into() } - pub(super) fn create(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn create(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::create_state_mut, CompletionHandle::into_create_state, @@ -316,17 +327,18 @@ impl Completion> { ) }, } + .into() } } impl Completion> { - pub(super) fn stat(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn stat(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::stat_state_mut, CompletionHandle::into_stat_state, @@ -335,17 +347,18 @@ impl Completion> { ) }, } + .into() } } impl Completion> { - pub(super) fn fallocate(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn fallocate(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::fallocate_state_mut, CompletionHandle::into_fallocate_state, @@ -354,15 +367,16 @@ impl Completion> { ) }, } + .into() } - pub(super) fn fsync(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn fsync(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::fsync_state_mut, CompletionHandle::into_fsync_state, @@ -371,15 +385,16 @@ impl Completion> { ) }, } + .into() } - pub(super) fn fdatasync(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn fdatasync(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::fdatasync_state_mut, CompletionHandle::into_fdatasync_state, @@ -388,16 +403,17 @@ impl Completion> { ) }, } + .into() } #[allow(unused)] - pub(super) fn noop(sim: Rc, key: usize) -> Self { - Self { - sim, + pub(super) fn noop(pending: Rc>, key: usize) -> Self { + CompletionInner::Poll { + pending, key, - poll: |sim, key, cx| { + poll: |pending, key, cx| { poll_completion( - sim, + pending, key, CompletionHandle::noop_state_mut, CompletionHandle::into_noop_state, @@ -406,6 +422,13 @@ impl Completion> { ) }, } + .into() + } +} + +impl From> for Completion { + fn from(inner: CompletionInner) -> Self { + Self { inner } } } @@ -413,21 +436,32 @@ impl Completion> { /// pending list, if it is present. /// /// If it is not present, then the future was polled to completion already. -impl Drop for Completion { +enum CompletionInner { + Poll { + pending: Rc>, + key: usize, + poll: fn(spin::MutexGuard<'_, PendingCompletions>, usize, &mut Context<'_>) -> Poll, + }, + Ready(Option), +} + +impl Drop for CompletionInner { fn drop(&mut self) { - self.sim.pending.lock().try_remove(self.key); + let Self::Poll { pending, key, .. } = self else { + return; + }; + pending.lock().try_remove(*key); } } fn poll_completion( - sim: &SimulatorInner, + mut pending: spin::MutexGuard<'_, PendingCompletions>, key: usize, state_mut: fn(&mut CompletionHandle) -> &mut CompletionState, into_state: fn(CompletionHandle) -> CompletionState, map: fn(S) -> T, cx: &mut Context<'_>, ) -> Poll { - let mut pending = sim.pending.lock(); match pending.get_mut(key) { None => unreachable!("completion polled after already complete"), Some(handle) => { @@ -450,12 +484,20 @@ fn poll_completion( } } +impl Unpin for Completion {} + impl Future for Completion { type Output = T; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); - (this.poll)(&this.sim, this.key, cx) + match &mut this.inner { + CompletionInner::Poll { pending, key, poll } => poll(pending.lock(), *key, cx), + CompletionInner::Ready(val) => match val.take() { + Some(val) => Poll::Ready(val), + None => Poll::Pending, + }, + } } } diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index f852ae5d809..0c733e5be7a 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, rc::Rc}; +use alloc::{boxed::Box, rc::Rc, sync::Arc}; use core::result::Result; use crate::{ @@ -38,6 +38,8 @@ pub enum Error { Cancelled, #[error("submission queue overflow")] SubmissionQueueOverflow, + #[error("too many pending completion futures")] + TooManyCompletions, } impl From for Error { @@ -109,61 +111,75 @@ impl SimulatorIO { fn submit( &self, sqe: Sqe, - completion: impl FnOnce(Rc, usize) -> Completion, + completion: impl FnOnce(Rc>, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>) -> CompletionHandle, - ) -> Completion { + ) -> Completion> { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - let mut state = CompletionState::Pending(None); - if let Err(_sqe) = executor.submit([sqe.attach(pending_entry.key())]) { - state = CompletionState::Ready(Err(Error::SubmissionQueueOverflow)); + match pending.vacant_entry() { + Some(pending_entry) => match executor.submit([sqe.attach(pending_entry.key())]) { + Err(_sqe) => Completion::ready(Err(Error::SubmissionQueueOverflow)), + Ok(()) => { + let key = pending_entry.key(); + pending_entry.insert(completion_handle(CompletionState::Pending(None))); + + completion(self.inner.pending.clone(), key) + } + }, + None => Completion::ready(Err(Error::TooManyCompletions)), } - let key = pending_entry.key(); - pending_entry.insert(completion_handle(state)); - - completion(self.inner.clone(), key) } fn submit_with( &self, sqe: Sqe, - completion: impl FnOnce(Rc, usize) -> Completion>, + completion: impl FnOnce(Rc>, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, ) -> Completion> { let mut executor = self.inner.executor.lock(); let mut pending = self.inner.pending.lock(); - let pending_entry = pending.vacant_entry(); - - let mut state = CompletionState::Pending(None); - if let Err(mut sqe) = executor.submit([sqe.attach(pending_entry.key())]) { - let buf = sqe - .next() - .expect("submitted one sqe therefore one must be returned on overflow") - .into_buf() - .expect("sqe must have been buffer-carrying"); - state = CompletionState::Ready(Err(ErrorWith { - error: Error::SubmissionQueueOverflow, - with: buf, - })); - } - let key = pending_entry.key(); - pending_entry.insert(completion_handle(state)); - completion(self.inner.clone(), key) + let reify = |sqe: Sqe| { + sqe.into_buf() + .map(|erased| erased.into_aligned()) + .expect("sqe must have been buffer-carrying") + }; + + match pending.vacant_entry() { + Some(pending_entry) => match executor.submit([sqe.attach(pending_entry.key())]) { + Err(mut sqe) => Completion::ready(Err(ErrorWith { + error: Error::SubmissionQueueOverflow, + with: reify( + sqe.next() + .expect("submitted one sqe therefore one must be returned on overflow"), + ), + })), + Ok(()) => { + let key = pending_entry.key(); + pending_entry.insert(completion_handle(CompletionState::Pending(None))); + + completion(self.inner.pending.clone(), key) + } + }, + None => Completion::ready(Err(ErrorWith { + error: Error::TooManyCompletions, + with: reify(sqe), + })), + } } } struct SimulatorInner { executor: spin::Mutex>, - pending: spin::Mutex, + pending: Rc>, } impl SimulatorInner { fn with_options(options: Options) -> Self { Self { - pending: spin::Mutex::new(PendingCompletions::with_capacity(options.cq_capacity())), + pending: Rc::new(spin::Mutex::new(PendingCompletions::with_capacity( + options.cq_capacity(), + ))), executor: spin::Mutex::new(Executor::new(options)), } } From 01cbcf1c775da850ce91e543ee79fa842f3bb180 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 16 Sep 2026 15:48:42 +0200 Subject: [PATCH 43/47] Make the completion future `Send` --- crates/runtime-io/src/buf.rs | 8 ++++++-- crates/runtime-io/src/sim/completion.rs | 26 ++++++++++++------------- crates/runtime-io/src/sim/mod.rs | 10 +++++----- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/crates/runtime-io/src/buf.rs b/crates/runtime-io/src/buf.rs index 7dbbbd911b7..636ff51b227 100644 --- a/crates/runtime-io/src/buf.rs +++ b/crates/runtime-io/src/buf.rs @@ -71,7 +71,7 @@ pub struct ErasedBox { impl ErasedBox { /// Create an [ErasedBox] from boxed [AlignedBytes].. - pub fn from_aligned(b: Box) -> Self { + pub fn from_aligned(b: Box) -> Self { let () = B::ASSERT_VALID_LAYOUT; let ptr = Box::into_raw(b); @@ -84,7 +84,7 @@ impl ErasedBox { } /// Reify `B` via casting. - pub fn into_aligned(self) -> Box { + pub fn into_aligned(self) -> Box { assert_eq!(self.len, size_of::()); assert_eq!(self.ty, TypeId::of::()); @@ -122,6 +122,10 @@ impl Drop for ErasedBox { } } +// SAFETY: [ErasedBox] is `Send` because it can only be constructed from a +// `Send` [AlignedBuffer]. +unsafe impl Send for ErasedBox {} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index 4b3f62a80ae..b6b06f68902 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -4,7 +4,7 @@ use core::{ task::{Context, Poll, Waker}, }; -use alloc::{boxed::Box, rc::Rc}; +use alloc::{boxed::Box, sync::Arc}; use slab::Slab; use crate::{ @@ -255,8 +255,8 @@ impl Completion { } } -impl Completion, ErrorWith>>> { - pub(super) fn write(pending: Rc>, key: usize) -> Self { +impl Completion, ErrorWith>>> { + pub(super) fn write(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -274,7 +274,7 @@ impl Completion, ErrorWith>, key: usize) -> Self { + pub(super) fn read(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -294,7 +294,7 @@ impl Completion, ErrorWith> { - pub(super) fn open(pending: Rc>, key: usize) -> Self { + pub(super) fn open(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -312,7 +312,7 @@ impl Completion> { .into() } - pub(super) fn create(pending: Rc>, key: usize) -> Self { + pub(super) fn create(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -332,7 +332,7 @@ impl Completion> { } impl Completion> { - pub(super) fn stat(pending: Rc>, key: usize) -> Self { + pub(super) fn stat(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -352,7 +352,7 @@ impl Completion> { } impl Completion> { - pub(super) fn fallocate(pending: Rc>, key: usize) -> Self { + pub(super) fn fallocate(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -370,7 +370,7 @@ impl Completion> { .into() } - pub(super) fn fsync(pending: Rc>, key: usize) -> Self { + pub(super) fn fsync(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -388,7 +388,7 @@ impl Completion> { .into() } - pub(super) fn fdatasync(pending: Rc>, key: usize) -> Self { + pub(super) fn fdatasync(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -407,7 +407,7 @@ impl Completion> { } #[allow(unused)] - pub(super) fn noop(pending: Rc>, key: usize) -> Self { + pub(super) fn noop(pending: Arc>, key: usize) -> Self { CompletionInner::Poll { pending, key, @@ -438,7 +438,7 @@ impl From> for Completion { /// If it is not present, then the future was polled to completion already. enum CompletionInner { Poll { - pending: Rc>, + pending: Arc>, key: usize, poll: fn(spin::MutexGuard<'_, PendingCompletions>, usize, &mut Context<'_>) -> Poll, }, @@ -501,7 +501,7 @@ impl Future for Completion { } } -fn reify( +fn reify( result: Result>, ) -> Result, ErrorWith>> { match result { diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 0c733e5be7a..1cad20a08df 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -111,7 +111,7 @@ impl SimulatorIO { fn submit( &self, sqe: Sqe, - completion: impl FnOnce(Rc>, usize) -> Completion>, + completion: impl FnOnce(Arc>, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>) -> CompletionHandle, ) -> Completion> { let mut executor = self.inner.executor.lock(); @@ -130,10 +130,10 @@ impl SimulatorIO { } } - fn submit_with( + fn submit_with( &self, sqe: Sqe, - completion: impl FnOnce(Rc>, usize) -> Completion>, + completion: impl FnOnce(Arc>, usize) -> Completion>, completion_handle: impl FnOnce(CompletionState>>) -> CompletionHandle, ) -> Completion> { let mut executor = self.inner.executor.lock(); @@ -171,13 +171,13 @@ impl SimulatorIO { struct SimulatorInner { executor: spin::Mutex>, - pending: Rc>, + pending: Arc>, } impl SimulatorInner { fn with_options(options: Options) -> Self { Self { - pending: Rc::new(spin::Mutex::new(PendingCompletions::with_capacity( + pending: Arc::new(spin::Mutex::new(PendingCompletions::with_capacity( options.cq_capacity(), ))), executor: spin::Mutex::new(Executor::new(options)), From 9d8183e63c95b72bd7b8c205a90914fe07cb0333 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 16 Sep 2026 15:59:46 +0200 Subject: [PATCH 44/47] Undo all changes in runtime-core --- crates/runtime-core/Cargo.toml | 3 +-- crates/runtime-core/src/lib.rs | 2 +- crates/runtime-core/src/sim/executor/mod.rs | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/runtime-core/Cargo.toml b/crates/runtime-core/Cargo.toml index 962883cb127..a3369a69f89 100644 --- a/crates/runtime-core/Cargo.toml +++ b/crates/runtime-core/Cargo.toml @@ -11,8 +11,7 @@ workspace = true [features] default = [] -alloc = [] -sim = ["alloc", "dep:async-task", "dep:spin"] +sim = ["dep:async-task", "dep:spin"] [dependencies] async-task = { version = "4.4", default-features = false, optional = true } diff --git a/crates/runtime-core/src/lib.rs b/crates/runtime-core/src/lib.rs index 4a89cdbe2b8..f7590ada98b 100644 --- a/crates/runtime-core/src/lib.rs +++ b/crates/runtime-core/src/lib.rs @@ -1,6 +1,6 @@ #![no_std] -#[cfg(any(feature = "sim", feature = "alloc"))] +#[cfg(feature = "sim")] extern crate alloc; #[cfg(test)] extern crate std; diff --git a/crates/runtime-core/src/sim/executor/mod.rs b/crates/runtime-core/src/sim/executor/mod.rs index a0fbca1bf7c..fbb7f7c0cf2 100644 --- a/crates/runtime-core/src/sim/executor/mod.rs +++ b/crates/runtime-core/src/sim/executor/mod.rs @@ -18,7 +18,7 @@ pub use task::{AbortHandle, JoinError, JoinHandle}; type Runnable = async_task::Runnable; -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RuntimeConfig { pub seed: u64, } From cc4c39bb23eddd013dc8b543ffbf09411ea5b32f Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Wed, 16 Sep 2026 16:42:24 +0200 Subject: [PATCH 45/47] Redo fs --- crates/runtime-io/src/lib.rs | 1 + crates/runtime-io/src/sim/executor.rs | 88 ++-- crates/runtime-io/src/sim/executor/sqe.rs | 64 +-- crates/runtime-io/src/sim/fs.rs | 517 +++++++++++++--------- crates/runtime-io/src/sim/mod.rs | 30 +- 5 files changed, 370 insertions(+), 330 deletions(-) diff --git a/crates/runtime-io/src/lib.rs b/crates/runtime-io/src/lib.rs index bf65f8a2392..4b13070fdc0 100644 --- a/crates/runtime-io/src/lib.rs +++ b/crates/runtime-io/src/lib.rs @@ -15,6 +15,7 @@ pub mod sim; /// Size in bytes of a disk sector. pub const SECTOR_SIZE: usize = 4096; +const SECTOR_SIZE64: u64 = SECTOR_SIZE as u64; /// Subset of the `statx` metadata. #[derive(Debug)] diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index c8f942b648f..276ec28636e 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,10 +1,6 @@ -use alloc::{ - boxed::Box, - collections::{BTreeMap, VecDeque}, - vec::Vec, -}; +use alloc::{collections::VecDeque, vec::Vec}; use core::{ - iter::{Chain, Map, Scan}, + iter::{Map, Scan}, num::NonZeroUsize, ops::Range, result::Result, @@ -14,7 +10,7 @@ use slab::Slab; use crate::{ sim::{completion::CompletionHandle, fs, Error}, - ErasedBox, ErrorWith, Statx, SECTOR_SIZE, + ErasedBox, ErrorWith, Statx, SECTOR_SIZE, SECTOR_SIZE64, }; pub use crate::sim::fs::Datasync; @@ -62,13 +58,13 @@ pub enum Operation { #[derive(Clone, Copy)] pub struct WriteSector { - pub page_offset: usize, + pub sector: usize, pub buf_offset: usize, } #[derive(Clone, Copy)] pub struct ReadSector { - pub page_offset: usize, + pub sector: usize, pub buf_offset: usize, } @@ -255,7 +251,7 @@ impl Results { } type ReadWriteOps = Scan, usize, fn(&mut usize, usize) -> Option>; -type SyncOps = Chain, fn(u64) -> Operation>, core::option::IntoIter>; +type SyncOps = Map Operation>; enum Pending { ReadWrite { ops: ReadWriteOps, results: Results }, @@ -413,7 +409,7 @@ pub struct Options { /// This basically limits how many [Sqe]s can be submitted in one batch. /// Should be a power of 2, or is otherwise rounded up to the next power of /// 2. - pub capacity: NonZeroUsize, + pub sq_capacity: NonZeroUsize, /// Override the completion queue capacity. /// /// By default, the completion queue's capacity is twice the submission @@ -431,11 +427,15 @@ pub struct Options { /// Should be a power of 2, or is otherwise rounded up to the next power of /// two. pub max_concurrency: NonZeroUsize, + /// Size in bytes of the virtual disk / filesystem. + /// + /// Should be a multiple of [SECTOR_SIZE] and is rounded if it isn't. + pub disk_space_bytes: u64, } impl Options { pub(crate) fn sq_capacity(&self) -> usize { - self.capacity.get().next_power_of_two() + self.sq_capacity.get().next_power_of_two() } pub(crate) fn cq_capacity(&self) -> usize { @@ -452,10 +452,11 @@ impl Options { impl Default for Options { fn default() -> Self { Self { - capacity: NonZeroUsize::new(8).unwrap(), + sq_capacity: NonZeroUsize::new(8).unwrap(), cq_capacity: None, cq_overflow: OnCqOverflow::default(), max_concurrency: NonZeroUsize::new(32).unwrap(), + disk_space_bytes: 2 * 4096, } } } @@ -469,7 +470,7 @@ pub struct Executor { // Scratch space for task selector select_executing: Vec, - fstree: BTreeMap, fs::File>, + fs: fs::Filesystem, cq_overflow: OnCqOverflow, cq_dropped: usize, @@ -479,13 +480,14 @@ impl Executor { pub fn new(options: Options) -> Self { let sq_capacity = options.sq_capacity(); let cq_capacity = options.cq_capacity(); + let fs_capacity = options.disk_space_bytes.next_multiple_of(SECTOR_SIZE64) as usize; Self { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), in_flight: Slab::with_capacity(2 * sq_capacity), executing: Vec::with_capacity(options.max_concurrency()), select_executing: Vec::with_capacity(options.max_concurrency()), - fstree: BTreeMap::new(), + fs: fs::Filesystem::new(fs_capacity), cq_overflow: options.cq_overflow, cq_dropped: 0, } @@ -502,10 +504,7 @@ impl Executor { self.in_flight.clear(); self.executing.clear(); self.cq_dropped = 0; - - for file in self.fstree.values_mut() { - file.power_loss(); - } + self.fs.power_loss(); } /// Restart the executor, simulating a process crash. @@ -757,15 +756,14 @@ impl Executor { else { unreachable!("invalid sqe: expected write") }; - let run = |WriteSector { - page_offset, - buf_offset, - }| { + let run = |WriteSector { sector, buf_offset }| { let bytes = buf.as_bytes(); let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - let buf = &buf.as_bytes()[buf_offset..end]; - fd.write_page(buf, page_offset as _).map_err(Into::into) + let buf: &[u8; SECTOR_SIZE] = buf.as_bytes()[buf_offset..end] + .try_into() + .expect("buffer must be sector aligned"); + fd.write_sector(buf, sector as _).map_err(Into::into) }; results.push(eff.traverse(run, Err)); results.is_complete() @@ -801,15 +799,14 @@ impl Executor { else { unreachable!("invalid sqe: expected read") }; - let run = |ReadSector { - page_offset, - buf_offset, - }| { + let run = |ReadSector { sector, buf_offset }| { let bytes = buf.as_bytes_mut(); let end = (buf_offset + SECTOR_SIZE).min(bytes.len()); - let buf = &mut buf.as_bytes_mut()[buf_offset..end]; - fd.read_page(buf, page_offset as _).map_err(Into::into) + let buf: &mut [u8; SECTOR_SIZE] = (&mut buf.as_bytes_mut()[buf_offset..end]) + .try_into() + .expect("buffer must be sector aligned"); + fd.read_sector(buf, sector as _).map_err(Into::into) }; results.push(eff.traverse(run, Err)); results.is_complete() @@ -845,10 +842,7 @@ impl Executor { else { unreachable!("invalid sqe: expected open") }; - let result = eff.traverse( - |()| self.fstree.get(&path).cloned().ok_or(Error::FileNotFound { path }), - Err, - ); + let result = eff.traverse(|()| self.fs.open(&path).map_err(Into::into), Err); let is_success = result.is_ok(); self.complete(Cqe { @@ -868,14 +862,7 @@ impl Executor { else { unreachable!("invalid sqe: expected create") }; - let run = |()| { - // Avoid cloning `path` if already exists. - if self.fstree.contains_key(&path) { - Err(Error::FileAlreadyExists { path }) - } else { - Ok(self.fstree.entry(path).or_default().clone()) - } - }; + let run = |()| self.fs.create(path).map_err(Into::into); let result = eff.traverse(run, Err); let is_success = result.is_ok(); self.complete(Cqe { @@ -914,7 +901,7 @@ impl Executor { else { unreachable!("invalid sqe: expected fallocate") }; - let result = eff.traverse(|()| fd.set_len(total_len).map_err(Into::into), Err); + let result = eff.traverse(|()| fd.reserve(total_len).map_err(Into::into), Err); let is_success = result.is_ok(); self.complete(Cqe { inner: CqeInner::Fallocate { result }, @@ -934,10 +921,7 @@ impl Executor { unreachable!("invalid sqe: expected fsync") }; let result = eff.traverse( - |FsyncEffect::Datasync(effect)| { - fd.fdatasync([effect]); - Ok(()) - }, + |FsyncEffect::Datasync(effect)| fd.apply_datasync(effect).map_err(Into::into), Err, ); results.push(result); @@ -974,13 +958,7 @@ impl Executor { else { unreachable!("invalid sqe: expected fdatasync") }; - let result = eff.traverse( - |effect| { - fd.fdatasync([effect]); - Ok(()) - }, - Err, - ); + let result = eff.traverse(|effect| fd.apply_datasync(effect).map_err(Into::into), Err); results.push(result); results.is_complete() }; diff --git a/crates/runtime-io/src/sim/executor/sqe.rs b/crates/runtime-io/src/sim/executor/sqe.rs index 8388b50db58..a5c130ac08a 100644 --- a/crates/runtime-io/src/sim/executor/sqe.rs +++ b/crates/runtime-io/src/sim/executor/sqe.rs @@ -6,7 +6,7 @@ use crate::{ fs::{self, Datasync}, Error, }, - ErasedBox, SECTOR_SIZE, + ErasedBox, SECTOR_SIZE, SECTOR_SIZE64, }; /// Opaque identifier of a scheduled [Sqe]. @@ -65,12 +65,12 @@ impl Sqe { } pub fn write(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - assert!(offset.is_multiple_of(SECTOR_SIZE as u64)); + assert!(offset.is_multiple_of(SECTOR_SIZE64)); Self::new(SqeInner::Write { fd, buf, offset }) } pub fn read(fd: fs::File, buf: ErasedBox, offset: u64) -> Self { - assert!(offset.is_multiple_of(SECTOR_SIZE as u64)); + assert!(offset.is_multiple_of(SECTOR_SIZE64)); Self::new(SqeInner::Read { fd, buf, offset }) } @@ -196,67 +196,49 @@ impl SqeInner { match self { SqeInner::Write { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; + let first_sector = (*offset / SECTOR_SIZE64) as usize; + let sector_count = buf_len / SECTOR_SIZE; Pending::ReadWrite { - ops: (0..page_count).scan(first_sector, |first_sector, page| { + ops: (0..sector_count).scan(first_sector, |first_sector, sector| { Some(Operation::WriteSector(WriteSector { - page_offset: *first_sector + page, - buf_offset: page * SECTOR_SIZE, + sector: *first_sector + sector, + buf_offset: sector * SECTOR_SIZE, })) }), - results: Results::new(page_count), + results: Results::new(sector_count), } } SqeInner::Read { buf, offset, .. } => { let buf_len = buf.as_bytes().len(); - let first_sector = (*offset / SECTOR_SIZE as u64) as usize; - let page_count = buf_len / SECTOR_SIZE; + let first_sector = (*offset / SECTOR_SIZE64) as usize; + let sector_count = buf_len / SECTOR_SIZE; Pending::ReadWrite { - ops: (0..page_count).scan(first_sector, |first_sector, page| { + ops: (0..sector_count).scan(first_sector, |first_sector, sector| { Some(Operation::ReadSector(ReadSector { - page_offset: *first_sector + page, - buf_offset: page * SECTOR_SIZE, + sector: *first_sector + sector, + buf_offset: sector * SECTOR_SIZE, })) }), - results: Results::new(page_count), + results: Results::new(sector_count), } } SqeInner::Fsync { fd } => { - let sector_count = fd.len() / SECTOR_SIZE as u64; - - let f = |offset: u64| -> Operation { - Operation::Fsync { - effect: FsyncEffect::Datasync(Datasync::Sector(offset)), - } - }; - + let sector_count = fd.len() / SECTOR_SIZE64; Pending::Sync { - ops: (0..sector_count) - .map(f as fn(u64) -> Operation) - .chain(Some(Operation::Fsync { - effect: Datasync::Length.into(), - })), + ops: fd.prepare_datasync().map( + (|effect| Operation::Fsync { + effect: FsyncEffect::Datasync(effect), + }) as fn(Datasync) -> Operation, + ), results: Results::new(1 + sector_count as usize), } } SqeInner::Fdatasync { fd } => { - let sector_count = fd.len() / SECTOR_SIZE as u64; - - let f = |offset: u64| -> Operation { - Operation::Fdatasync { - effect: Datasync::Sector(offset), - } - }; - + let sector_count = fd.len() / SECTOR_SIZE64; Pending::Sync { - ops: (0..sector_count) - .map(f as fn(u64) -> Operation) - .chain(Some(Operation::Fdatasync { - effect: Datasync::Length, - })), + ops: fd.prepare_datasync().map(|effect| Operation::Fdatasync { effect }), results: Results::new(1 + sector_count as usize), } } diff --git a/crates/runtime-io/src/sim/fs.rs b/crates/runtime-io/src/sim/fs.rs index 3c73f0525cc..5d1c6094b20 100644 --- a/crates/runtime-io/src/sim/fs.rs +++ b/crates/runtime-io/src/sim/fs.rs @@ -1,295 +1,378 @@ -use alloc::{collections::BTreeMap, sync::Arc}; -use core::{ - fmt, - sync::atomic::{AtomicU64, Ordering}, -}; +use alloc::{boxed::Box, collections::BTreeMap, sync::Arc, vec::Vec}; +use core::mem; -pub const PAGE_SIZE: usize = 4096; -const PAGE_SIZE_U64: u64 = PAGE_SIZE as u64; +use spin::Mutex; + +use crate::{SECTOR_SIZE, SECTOR_SIZE64}; + +type SectorId = usize; +type FileId = usize; #[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] pub enum Error { - #[error("unaligned offset")] - UnalignedOffset, - #[error("unaligned buffer")] - UnalignedBuffer, - #[error("offset overflow")] - OffsetOverflow, + #[error("file already exists")] + FileAlreadyExists, + #[error("file not found")] + FileNotFound, + #[error("no space left on device")] + NoSpace, + #[error("invalid argument")] + InvalidArgument, } pub type Result = core::result::Result; -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -struct PageIndex(u64); +#[derive(Debug)] +struct Sector { + volatile: Box<[u8; SECTOR_SIZE]>, + durable: Box<[u8; SECTOR_SIZE]>, +} + +#[derive(Debug)] +struct FileState { + // Logical sector index -> physical sector. + sectors: Vec, -impl PageIndex { - fn from_offset(offset: u64) -> Self { - assert!(offset.is_multiple_of(PAGE_SIZE_U64)); - Self(offset / PAGE_SIZE_U64) - } + volatile_len: u64, + durable_len: u64, + + // Incremented whenever a datasync snapshot is created. + dirty_generation: u64, + // Logical sector -> generation it was last dirtied. + dirty_sectors: BTreeMap, + // Generation in which the length was last dirtied. + dirty_len: Option, } -struct Page { - bytes: spin::Mutex<[u8; PAGE_SIZE]>, +/// A virtual filesystem that keeps track of files, allocated space and +/// name->file mappings. +#[derive(Clone, Debug)] +pub struct Filesystem { + inner: Arc>, } -impl Page { - fn zeroed() -> Self { - Self { - bytes: spin::Mutex::new([0; PAGE_SIZE]), - } - } +#[derive(Debug)] +struct FsInner { + sectors: Vec, + free: Vec, + + files: Vec, + paths: BTreeMap, FileId>, } -#[derive(Default)] -struct PageMap { - volatile: BTreeMap>, - durable: BTreeMap>, +/// A virtual file in the [Filesystem]. +#[derive(Clone, Debug)] +pub struct File { + fs: Filesystem, + id: FileId, } -impl PageMap { - /// Reset the volatile to the durable state. - fn power_loss(&mut self) { - self.volatile.clear(); - for (idx, page) in &self.durable { - self.volatile.insert(*idx, Arc::clone(page)); - } - } +/// Individual effects produced by [File::prepare_datasync]. +/// +/// `fsync` / `fdatasync` is modelled as a series of sector effects and +/// potentially the file length. This allows to inject failures, in particular +/// partial failure of a sync operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Datasync { + Sector { sector: usize, generation: u64 }, + Length { generation: u64 }, +} - /// Move the page at `index` from the volatile to the durable state. - fn sync(&mut self, index: PageIndex) { - self.durable.insert(index, self.volatile.get(&index).cloned().unwrap()); - } +impl Filesystem { + pub fn new(capacity: usize) -> Self { + assert!(capacity.is_multiple_of(SECTOR_SIZE)); + + let sector_count = capacity / SECTOR_SIZE; + let sectors = (0..sector_count) + .map(|_| Sector { + volatile: Box::new([0; SECTOR_SIZE]), + durable: Box::new([0; SECTOR_SIZE]), + }) + .collect(); - /// Get the page at `index` for reading. Uses the volatile state. - fn readonly_page(&self, index: PageIndex) -> Option> { - self.volatile.get(&index).cloned() + Self { + inner: Arc::new(Mutex::new(FsInner { + sectors, + free: (0..sector_count).rev().collect(), + files: Vec::new(), + paths: BTreeMap::new(), + })), + } } - /// Get the page at `index` for writing, or allocate a new page. - /// Uses the volatile state. - fn writable_page(&mut self, index: PageIndex) -> Arc { - let page = self.volatile.entry(index).or_insert_with(|| Arc::new(Page::zeroed())); + pub fn create(&self, path: Box) -> Result { + let mut fs = self.inner.lock(); - // Copy-on-write if the page is in the durable state. - if self - .durable - .get(&index) - .is_some_and(|durable| Arc::ptr_eq(durable, page)) - { - let bytes = *page.bytes.lock(); - *page = Arc::new(Page { - bytes: spin::Mutex::new(bytes), - }); + if fs.paths.contains_key(&path) { + return Err(Error::FileAlreadyExists); } - Arc::clone(page) - } + let id = fs.files.len(); + + fs.files.push(FileState { + sectors: Vec::new(), + volatile_len: 0, + durable_len: 0, + dirty_generation: 0, + dirty_sectors: BTreeMap::new(), + dirty_len: None, + }); - /// Change the allocated space, allocating or deallocating pages as needed. - /// Changes the volatile state only. - fn set_len_volatile(&mut self, old_len: u64, new_len: u64) { - Self::set_len(&mut self.volatile, old_len, new_len); + fs.paths.insert(path, id); + + Ok(File { fs: self.clone(), id }) } - /// Like [Self::set_len_volatile], but operate on the durable state only. - fn set_len_durable(&mut self, old_len: u64, new_len: u64) { - Self::set_len(&mut self.durable, old_len, new_len); + pub fn open(&self, path: &str) -> Result { + let fs = self.inner.lock(); + + let id = *fs.paths.get(path).ok_or(Error::FileNotFound)?; + + Ok(File { fs: self.clone(), id }) } - fn set_len(page_map: &mut BTreeMap>, old_len: u64, new_len: u64) { - use core::cmp::Ordering::*; + /// Simulate power loss. + /// + /// All unsynced data and metadata are discarded. + pub fn power_loss(&self) { + let mut fs = self.inner.lock(); - match new_len.cmp(&old_len) { - Equal => {} - Greater => { - let first_new_page = old_len / PAGE_SIZE_U64; - let end_page = new_len / PAGE_SIZE_U64; + for file_id in 0..fs.files.len() { + let durable_len = fs.files[file_id].durable_len; + let durable_sectors = sector_count(durable_len); - for index in first_new_page..end_page { - page_map - .entry(PageIndex(index)) - .or_insert_with(|| Arc::new(Page::zeroed())); - } + // Allocations beyond the durable file length were not durably + // reachable, so they become free again. + while fs.files[file_id].sectors.len() > durable_sectors { + let sector_id = fs.files[file_id].sectors.pop().unwrap(); + fs.free.push(sector_id); } - Less => { - let first_removed = PageIndex::from_offset(new_len); - page_map.retain(|&index, _| index < first_removed); + + let sector_ids = fs.files[file_id].sectors.clone(); + + for sector_id in sector_ids { + let sector = &mut fs.sectors[sector_id]; + sector.volatile.copy_from_slice(&*sector.durable); } + + let file = &mut fs.files[file_id]; + file.volatile_len = file.durable_len; + file.dirty_generation = 0; + file.dirty_sectors.clear(); + file.dirty_len = None; } } } -#[derive(Clone, Copy)] -pub enum Datasync { - Sector(u64), - Length, -} - -/// A memory-backed file. -/// -/// A [File] is backed by a sparse array of [Page]s. Missing pages are read as -/// zeroes. -/// -/// Read and write operations must be page-aligned. Only full pages can be read -/// or written. Writing a page is atomic. -#[derive(Clone, Debug, Default)] -pub struct File { - inner: Arc, -} - impl File { - pub fn power_loss(&self) { - self.inner.power_loss(); + pub(super) fn len(&self) -> u64 { + self.fs.inner.lock().files[self.id].volatile_len } - pub fn len(&self) -> u64 { - self.inner.len() + /// Preallocate enough physical sectors to cover `new_len`. + /// + /// Growing fails with [Error::NoSpace] if the fixed filesystem pool cannot + /// satisfy the allocation. + /// + /// Shrinking is deliberately not handled by this operation. + /// + /// The result of this operation is not durable until [Self::apply_datasync] + /// is called. + pub(super) fn reserve(&self, new_len: u64) -> Result<()> { + let mut fs = self.fs.inner.lock(); + grow(&mut fs, self.id, new_len) } - pub fn is_empty(&self) -> bool { - self.inner.is_empty() - } + /// Read one complete sector. + /// + /// Returns 0 at or beyond EOF. Reading one sector is atomic. + pub(super) fn read_sector(&self, dst: &mut [u8; SECTOR_SIZE], sector: usize) -> Result { + let fs = self.fs.inner.lock(); + let file = &fs.files[self.id]; - pub fn read_page(&self, dst: &mut [u8], index: u64) -> Result { - self.inner.read_page(dst, index) - } + let offset = sector as u64 * SECTOR_SIZE64; + if offset >= file.volatile_len { + return Ok(0); + } - pub fn write_page(&self, src: &[u8], index: u64) -> Result { - self.inner.write_page(src, index) - } + let sector_id = file.sectors[sector]; + dst.copy_from_slice(&fs.sectors[sector_id].volatile[..]); - pub fn fdatasync(&self, ops: impl IntoIterator) { - self.inner.fdatasync(ops); + Ok(SECTOR_SIZE) } - pub fn set_len(&self, new_len: u64) -> Result<()> { - self.inner.set_len(new_len) - } -} + /// Write one complete sector. + /// + /// Like `pwrite`, this may extend the file. Extending can fail with + /// [Error::NoSpace]. Writing one sector is atomic. + pub(super) fn write_sector(&self, src: &[u8; SECTOR_SIZE], sector: usize) -> Result { + let mut fs = self.fs.inner.lock(); -#[derive(Default)] -struct FileInner { - pages: spin::Mutex, - volatile_len: AtomicU64, - durable_len: AtomicU64, -} + let end = (sector + 1).checked_mul(SECTOR_SIZE).ok_or(Error::InvalidArgument)? as u64; + grow(&mut fs, self.id, end)?; -impl fmt::Debug for FileInner { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FileInner") - .field("volatile_len", &self.volatile_len) - .field("durable_len", &self.durable_len) - .finish() - } -} + let sector_id = fs.files[self.id].sectors[sector]; -impl FileInner { - /// Simulate a crash by resetting to the durable state. - fn power_loss(&self) { - self.volatile_len - .store(self.durable_len.load(Ordering::Relaxed), Ordering::Relaxed); - self.pages.lock().power_loss(); - } + fs.sectors[sector_id].volatile.copy_from_slice(src); + let generation = fs.files[self.id].dirty_generation; + fs.files[self.id].dirty_sectors.insert(sector, generation); - fn len(&self) -> u64 { - self.volatile_len.load(Ordering::Relaxed) + Ok(SECTOR_SIZE) } - #[allow(unused)] - fn is_empty(&self) -> bool { - self.len() == 0 + /// Produce a series of [Datasync] effects to model `fdatasync`. + /// + /// The iterator captures sectors that are marked dirty at the time this + /// method is called, and ignores sectors dirtied later (while traversing + /// the iterator). Similarly for the file length. + /// + /// This is not an accurate model of how `fdatasync` works, but allows + /// interleaving of effects and injection of faults to produce + /// partially-durable states. + /// + /// [Datasync] effects are executed via [Self::apply_datasync]. + pub(super) fn prepare_datasync(&self) -> IterDatasync { + let mut fs = self.fs.inner.lock(); + let file = &mut fs.files[self.id]; + + let generation = file.dirty_generation; + file.dirty_generation += 1; + + IterDatasync { + file: self.clone(), + generation, + next_sector: 0, + length_pending: true, + } } - /// Change the file length. - /// - /// The new length must be page-aligned. + /// Execute one [Datasync] effect produced by [Self::prepare_datasync]. /// - /// Extending allocates pages eagerly as needed. Shrinking drops all pages - /// at or beyond the new EOF. - fn set_len(&self, new_len: u64) -> Result<()> { - if !new_len.is_multiple_of(PAGE_SIZE_U64) { - return Err(Error::UnalignedOffset); + /// A sector is removed from the dirty set only after it succeeds. + pub(super) fn apply_datasync(&self, effect: Datasync) -> Result<()> { + let mut fs = self.fs.inner.lock(); + + match effect { + Datasync::Sector { sector, generation } => { + // The sector may cease to exist if the file was concurrently + // truncated (not currently supported). Treat such an obsolete + // sync effect as already satisfied. + let Some(§or_id) = fs.files[self.id].sectors.get(sector) else { + fs.files[self.id].dirty_sectors.remove(§or); + return Ok(()); + }; + + // Persist what is visible when this effect executes. + let volatile = fs.sectors[sector_id].volatile.clone(); + fs.sectors[sector_id].durable.copy_from_slice(&*volatile); + + // Clear dirty flag only if the sector hasn't been modified + // since. + let file = &mut fs.files[self.id]; + if file + .dirty_sectors + .get(§or) + .is_some_and(|&dirty_generation| dirty_generation <= generation) + { + file.dirty_sectors.remove(§or); + } + } + + Datasync::Length { generation } => { + let volatile_len = fs.files[self.id].volatile_len; + let file = &mut fs.files[self.id]; + // Persist the length visible when this effect executes. + file.durable_len = volatile_len; + // Clear dirty flag only if no resize happened since. + if file + .dirty_len + .is_some_and(|dirty_generation| dirty_generation <= generation) + { + file.dirty_len = None; + } + } } - self.pages - .lock() - .set_len_volatile(self.volatile_len.load(Ordering::Relaxed), new_len); - self.volatile_len.store(new_len, Ordering::Relaxed); Ok(()) } +} - /// Read one complete page. - fn read_page(&self, dst: &mut [u8], index: u64) -> Result { - if dst.len() != PAGE_SIZE { - return Err(Error::UnalignedBuffer); - } +pub(super) struct IterDatasync { + file: File, + generation: u64, + next_sector: usize, + length_pending: bool, +} - let offset = index.checked_mul(PAGE_SIZE as u64).ok_or(Error::OffsetOverflow)?; - let len = self.volatile_len.load(Ordering::Relaxed); - if offset >= len { - return Ok(0); - } +impl Iterator for IterDatasync { + type Item = Datasync; - match self.get_page(PageIndex(index)) { - Some(page) => { - dst.copy_from_slice(&*page.bytes.lock()); - } - None => { - dst.fill(0); - } - } + fn next(&mut self) -> Option { + let fs = self.file.fs.inner.lock(); + let file = &fs.files[self.file.id]; - Ok(PAGE_SIZE) - } + if let Some((§or, _)) = file + .dirty_sectors + .range(self.next_sector..) + .find(|(_, generation)| **generation <= self.generation) + { + self.next_sector = sector + 1; + return Some(Datasync::Sector { + sector, + generation: self.generation, + }); + } - /// Write one complete page. - fn write_page(&self, src: &[u8], index: u64) -> Result { - if src.len() != PAGE_SIZE { - return Err(Error::UnalignedBuffer); + if mem::take(&mut self.length_pending) && file.dirty_len.is_some_and(|generation| generation <= self.generation) + { + return Some(Datasync::Length { + generation: self.generation, + }); } - let end = index - .checked_add(1) - .and_then(|pages| pages.checked_mul(PAGE_SIZE_U64)) - .ok_or(Error::OffsetOverflow)?; + None + } +} - let page = self.get_or_allocate_page(PageIndex(index)); - page.bytes.lock().copy_from_slice(src); +fn sector_count(len: u64) -> usize { + len.div_ceil(SECTOR_SIZE64) as usize +} - self.volatile_len.fetch_max(end, Ordering::Relaxed); +fn grow(fs: &mut FsInner, file_id: FileId, new_len: u64) -> Result<()> { + let old_len = fs.files[file_id].volatile_len; - Ok(src.len()) + if new_len < old_len { + return Err(Error::InvalidArgument); } - /// Execute an `fdatasync(2)` operation as a series of [Datasync] effects. - /// - /// The result may or may not leave the durable state in the same state as - /// the volatile state at the time the operation started. - /// - /// It is the caller's responsibility to decide whether the operation is - /// considered successful - a partial operation may report success, or a - /// complete operation may report failure. - fn fdatasync(&self, ops: impl IntoIterator) { - for op in ops { - match op { - Datasync::Sector(offset) => { - self.pages.lock().sync(PageIndex(offset)); - } - Datasync::Length => { - let new_durable_len = self.volatile_len.load(Ordering::Relaxed); - let old_durable_len = self.durable_len.swap(new_durable_len, Ordering::Relaxed); - self.pages.lock().set_len_durable(old_durable_len, new_durable_len); - } - } - } + if new_len == old_len { + return Ok(()); } - fn get_page(&self, index: PageIndex) -> Option> { - self.pages.lock().readonly_page(index) + let old_sector_count = sector_count(old_len); + let new_sector_count = sector_count(new_len); + let needed = new_sector_count - old_sector_count; + + // Make growth atomic with respect to ENOSPC: either all required sectors + // are allocated, or nothing changes. + if fs.free.len() < needed { + return Err(Error::NoSpace); } - fn get_or_allocate_page(&self, index: PageIndex) -> Arc { - self.pages.lock().writable_page(index) + for _ in 0..needed { + let sector_id = fs.free.pop().unwrap(); + + // Newly allocated filesystem space reads as zero. Clearing both copies + // also prevents data from a previous owner from becoming observable. + fs.sectors[sector_id].volatile.fill(0); + fs.sectors[sector_id].durable.fill(0); + + fs.files[file_id].sectors.push(sector_id); } + + let file = &mut fs.files[file_id]; + file.volatile_len = new_len; + file.dirty_len = Some(file.dirty_generation); + + Ok(()) } diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 1cad20a08df..3df2458742e 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -23,10 +23,6 @@ pub use crate::{ #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("file not found")] - FileNotFound { path: Box }, - #[error("file already exists")] - FileAlreadyExists { path: Box }, #[error("failed to write expected number of bytes")] ShortWrite { expected: usize, written: usize }, #[error("unexpected eof")] @@ -176,11 +172,11 @@ struct SimulatorInner { impl SimulatorInner { fn with_options(options: Options) -> Self { + let pending = PendingCompletions::with_capacity(options.cq_capacity()); + let executor = Executor::new(options); Self { - pending: Arc::new(spin::Mutex::new(PendingCompletions::with_capacity( - options.cq_capacity(), - ))), - executor: spin::Mutex::new(Executor::new(options)), + pending: Arc::new(spin::Mutex::new(pending)), + executor: spin::Mutex::new(executor), } } } @@ -255,6 +251,8 @@ impl SpacetimeIO for SimulatorIO { mod tests { use spacetimedb_runtime_core::sim::Rng; + use crate::SECTOR_SIZE64; + use super::*; impl TaskSelector for Rng { @@ -359,7 +357,7 @@ mod tests { buf.clear(); buf }; - let buf = rt.run(|io| io.read_exact_at(fd, buf, SECTOR_SIZE as u64)).unwrap(); + let buf = rt.run(|io| io.read_exact_at(fd, buf, SECTOR_SIZE64)).unwrap(); assert_eq!(buf.0, [4; SECTOR_SIZE]); } @@ -369,7 +367,7 @@ mod tests { let rt = Runtime::new(); let fd = rt.run(|io| io.create_file("/data/test".into())).unwrap(); - rt.run(|io| io.reserve(fd.clone(), 2 * SECTOR_SIZE as u64)).unwrap(); + rt.run(|io| io.reserve(fd.clone(), 2 * SECTOR_SIZE64)).unwrap(); // Check that reserved space reads as zeroes. let buf = rt @@ -379,15 +377,13 @@ mod tests { // The length is reported as the preallocated length. let stat = rt.run(|io| io.statx(fd.clone())).unwrap(); - assert_eq!(stat.size, 2 * SECTOR_SIZE as u64); + assert_eq!(stat.size, 2 * SECTOR_SIZE64); // Overwriting the second sector works. let buf = rt - .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([42; SECTOR_SIZE])), SECTOR_SIZE as u64)) - .unwrap(); - let buf = rt - .run(|io| io.read_exact_at(fd.clone(), buf, SECTOR_SIZE as u64)) + .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([42; SECTOR_SIZE])), SECTOR_SIZE64)) .unwrap(); + let buf = rt.run(|io| io.read_exact_at(fd.clone(), buf, SECTOR_SIZE64)).unwrap(); assert_eq!(buf.0, [42; SECTOR_SIZE]); // The first sector still reads as zeroes. let buf = rt.run(|io| io.read_exact_at(fd, buf, 0)).unwrap(); @@ -400,7 +396,7 @@ mod tests { matches!( rt.run(|io| io.open_file("/data/test".into())), - Err(Error::FileNotFound { .. }) + Err(Error::Fs(fs::Error::FileNotFound)) ); rt.run(|io| io.create_file("/data/test".into())).unwrap(); assert!(rt.run(|io| io.open_file("/data/test".into())).is_ok()); @@ -420,7 +416,7 @@ mod tests { rt.run(|io| io.fdatasync(fd.clone())).unwrap(); let mut buf = rt - .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([2; SECTOR_SIZE])), SECTOR_SIZE as u64)) + .run(|io| io.write_all_at(fd.clone(), Box::new(Buf([2; SECTOR_SIZE])), SECTOR_SIZE64)) .map_err(ErrorWith::into_err) .unwrap(); buf.clear(); From 5dc8ace0aa43fd1c212525cd17c56682f3957349 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 17 Sep 2026 15:22:15 +0200 Subject: [PATCH 46/47] Cancel and complete CQEs on restart --- crates/runtime-io/src/sim/completion.rs | 2 +- crates/runtime-io/src/sim/executor.rs | 33 +++++++++----- crates/runtime-io/src/sim/mod.rs | 58 +++++++++++++++---------- 3 files changed, 58 insertions(+), 35 deletions(-) diff --git a/crates/runtime-io/src/sim/completion.rs b/crates/runtime-io/src/sim/completion.rs index b6b06f68902..7064a5d3f26 100644 --- a/crates/runtime-io/src/sim/completion.rs +++ b/crates/runtime-io/src/sim/completion.rs @@ -478,7 +478,7 @@ fn poll_completion( match state { CompletionState::Ready(result) => Poll::Ready((map)(result)), - _ => unreachable!(), + CompletionState::Pending(_) => unreachable!("pending case already handled"), } } } diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index 276ec28636e..de0f7dfe97e 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -173,6 +173,10 @@ impl InFlight { fn is_active(&self) -> bool { self.state.is_active() } + + fn cancel(self) -> Cqe { + self.sqe.cancel(self.user_data) + } } enum InFlightState { @@ -509,18 +513,24 @@ impl Executor { /// Restart the executor, simulating a process crash. /// - /// Unlike [Self::crash], this will drive the currently executing operations - /// to completion. Submissions that were not yet scheduled are dropped. The - /// file state remains unchanged. + /// Unlike [Self::crash], this will allow the currently executing operations + /// to complete. It will, however, cancel all in-flight submissions, even if + /// they were partly completed. + /// + /// Submissions that were not yet scheduled are dropped. The file state + /// remains unchanged. /// /// Execution is subject to `faults`. If a fault evaluates to [Fault::Skip], /// that operation is dropped. /// - /// After this method returns, the completion queue is empty. - pub fn restart(&mut self, faults: &mut impl FaultInjector) { + /// All outstanding [Cqe]s are fed to `complete`, including completions that + /// happen as part of the shutdown. The completion queue is empty after this + /// method returns. + pub fn restart(&mut self, faults: &mut impl FaultInjector, mut complete: impl FnMut(Cqe)) { self.submissions.clear(); - let cq_overflow_orig = self.cq_overflow; - self.cq_overflow = OnCqOverflow::Drop; + self.cq_dropped = 0; + + self.completed().for_each(&mut complete); while let Some(op) = self.executing.pop() { if let Some(in_flight) = self.in_flight.get(op.sqe.key()) && !in_flight.is_active() @@ -528,11 +538,12 @@ impl Executor { continue; } self.execute_op(op, faults); + self.completed().for_each(&mut complete); + } + self.executing.clear(); + for in_flight in self.in_flight.drain() { + complete(in_flight.cancel()); } - self.in_flight.clear(); - self.completions.clear(); - self.cq_overflow = cq_overflow_orig; - self.cq_dropped = 0; } /// Submit a batch of [Sqe]s for later execution. diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 3df2458742e..9fad5b48f2a 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -1,8 +1,11 @@ use alloc::{boxed::Box, rc::Rc, sync::Arc}; -use core::result::Result; +use core::{result::Result, task::Waker}; use crate::{ - sim::completion::{CompletionState, PendingCompletions}, + sim::{ + completion::{CompletionState, PendingCompletions}, + executor::Cqe, + }, AlignedBytes, ErasedBox, ErrorWith, ReadWriteResult, SpacetimeIO, Statx, }; @@ -60,31 +63,36 @@ impl SimulatorIO { let mut executor = self.inner.executor.lock(); let mut progress = executor.tick(task_selector, faults); - executor - .completed() - .map(|cqe| { - let key = cqe.user_data().expect("user data must be set"); - let mut pending = self.inner.pending.lock(); - // If the handle is no longer present in `pending`, the - // completion future was dropped. - let handle = pending.get_mut(key)?; - cqe.complete(handle) - }) - .for_each(|waker| { - if let Some(waker) = waker { - waker.wake(); - } - progress |= true - }); + executor.completed().for_each(|cqe| { + self.process_cqe(cqe); + progress |= true; + }); progress } + fn process_cqe(&self, cqe: Cqe) { + let key = cqe.user_data().expect("user data must be set"); + let maybe_waker = (|| -> Option { + let mut pending = self.inner.pending.lock(); + // If the handle is no longer present in `pending`, the + // completion future was dropped. + let handle = pending.get_mut(key)?; + cqe.complete(handle) + })(); + if let Some(waker) = maybe_waker { + waker.wake(); + } + } + /// Simulate a power loss event. /// /// All submitted and executing operations are cancelled, and files reset to - /// their durable state. Completions that have not been signalled will be - /// dropped, too. + /// their durable state. + /// + /// The caller must uphold that no pending [Completion] futures are live (as + /// would happen during an actual power loss event). Polling a [Completion] + /// future after calling this method will panic. pub fn power_loss(&self) { self.inner.executor.lock().power_loss(); self.inner.pending.lock().clear(); @@ -98,10 +106,14 @@ impl SimulatorIO { /// Submissions that were not yet scheduled are dropped. The file state /// remains unchanged. /// - /// Completions that were not signalled during shutdown are dropped. + /// Pending [Completion]s will resolve to [Error::Cancelled]. + /// + /// Note that the simulator owns the storage for [Completion]s until they + /// are either dropped or completed. So they count toward the completion + /// queue capacity. To simulate an actual process crash, the caller should + /// prefer to drop pending [Completion]s. pub fn restart(&self, faults: &mut impl FaultInjector) { - self.inner.executor.lock().restart(faults); - self.inner.pending.lock().clear(); + self.inner.executor.lock().restart(faults, |cqe| self.process_cqe(cqe)) } fn submit( From fd2bc88e2295a82b1db2695fe657bb1cf4f784d4 Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Fri, 18 Sep 2026 10:01:45 +0200 Subject: [PATCH 47/47] Revisit task selection and document fault injection. --- crates/runtime-io/src/sim/executor.rs | 188 ++++++------------------ crates/runtime-io/src/sim/faults.rs | 198 ++++++++++++++++++++++++++ crates/runtime-io/src/sim/mod.rs | 41 +++++- 3 files changed, 276 insertions(+), 151 deletions(-) create mode 100644 crates/runtime-io/src/sim/faults.rs diff --git a/crates/runtime-io/src/sim/executor.rs b/crates/runtime-io/src/sim/executor.rs index de0f7dfe97e..165bfd8cb03 100644 --- a/crates/runtime-io/src/sim/executor.rs +++ b/crates/runtime-io/src/sim/executor.rs @@ -1,4 +1,4 @@ -use alloc::{collections::VecDeque, vec::Vec}; +use alloc::collections::VecDeque; use core::{ iter::{Map, Scan}, num::NonZeroUsize, @@ -9,27 +9,19 @@ use core::{ use slab::Slab; use crate::{ - sim::{completion::CompletionHandle, fs, Error}, + sim::{ + completion::CompletionHandle, + faults::{EitherOrBoth, IndexSelector, TaskSelection, TaskSelector}, + fs::{self, Datasync}, + Error, FaultInjector, + }, ErasedBox, ErrorWith, Statx, SECTOR_SIZE, SECTOR_SIZE64, }; -pub use crate::sim::fs::Datasync; - mod sqe; use sqe::SqeInner; pub use sqe::{LinkKind, Sqe, SqeId}; -pub trait TaskSelector { - /// Deterministically select zero or more tasks to advance. - /// - /// `task_count` is the number of currently outstanding tasks. The returned - /// iterator must return indexes in the range `0..task_count` and not yield - /// duplicate elements. - /// - /// Called once per [Executor::tick]. - fn select_tasks(&self, task_count: usize) -> impl IntoIterator; -} - // TODO: There is no difference between fsync and fdatasync until we extend // [Statx] with additional fields. #[derive(Clone, Copy)] @@ -287,111 +279,6 @@ impl Executing { } } -pub enum Fault { - /// Drop the operation entirely. - /// - /// Note that this is not generally possible in `io-uring`: an SQE always - /// yields a CQE, even if it was cancelled or returned an error. It may be - /// useful occasionally to construct "byzantine" failures. - Skip, - /// Put the operation back onto the queue for later execution. - Delay(T), - /// Execute a visible effect. - Visible(Effect), -} - -impl Fault { - fn exec_visible(self, f: impl FnOnce(EitherOrBoth)) -> Option { - match self { - Fault::Skip => None, - Fault::Delay(effect) => Some(effect), - Fault::Visible(visible) => { - visible.exec(f); - None - } - } - } -} - -pub enum Effect { - /// Run the operation as normal. - Run(T), - /// Run the effect, but report an injected error. - RunThenError { effect: T, error: Error }, - /// Skip the effect, but report an injected error. - SkipThenError { error: Error }, -} - -impl Effect { - fn exec(self, f: impl FnOnce(EitherOrBoth)) { - use EitherOrBoth::*; - match self { - Effect::Run(effect) => f(Left(effect)), - Effect::RunThenError { effect, error } => f(Both(effect, error)), - Effect::SkipThenError { error } => f(Right(error)), - } - } -} - -enum EitherOrBoth { - Left(T), - Right(U), - Both(T, U), -} - -impl EitherOrBoth { - fn traverse(self, f: impl FnOnce(T) -> V, g: impl FnOnce(U) -> V) -> V { - match self { - Self::Left(t) => f(t), - Self::Right(u) => g(u), - Self::Both(t, u) => { - f(t); - g(u) - } - } - } -} - -pub trait FaultInjector { - fn inject_write_sector_fault(&mut self, _: &InFlight, op: WriteSector) -> Fault { - Fault::Visible(Effect::Run(op)) - } - - fn inject_read_sector_fault(&mut self, _: &InFlight, op: ReadSector) -> Fault { - Fault::Visible(Effect::Run(op)) - } - - fn inject_open_fault(&mut self, _: &InFlight) -> Fault<()> { - Fault::Visible(Effect::Run(())) - } - - fn inject_create_fault(&mut self, _: &InFlight) -> Fault<()> { - Fault::Visible(Effect::Run(())) - } - - fn inject_stat_fault(&mut self, _: &InFlight) -> Fault<()> { - Fault::Visible(Effect::Run(())) - } - - fn inject_fallocate_fault(&mut self, _: &InFlight) -> Fault<()> { - Fault::Visible(Effect::Run(())) - } - - fn inject_fsync_fault(&mut self, _: &InFlight, op: FsyncEffect) -> Fault { - Fault::Visible(Effect::Run(op)) - } - - fn inject_fdatasync_fault(&mut self, _: &InFlight, op: Datasync) -> Fault { - Fault::Visible(Effect::Run(op)) - } - - fn inject_noop_fault(&mut self, _: &InFlight) -> Fault<()> { - Fault::Visible(Effect::Run(())) - } -} - -impl FaultInjector for () {} - /// Completion queue overflow policy. /// /// Note that we do **not** model `IORING_FEAT_NODROP`, because we never want @@ -470,9 +357,7 @@ pub struct Executor { completions: VecDeque>, in_flight: Slab>, - executing: Vec, - // Scratch space for task selector - select_executing: Vec, + executing: VecDeque, fs: fs::Filesystem, @@ -489,8 +374,7 @@ impl Executor { submissions: VecDeque::with_capacity(sq_capacity), completions: VecDeque::with_capacity(cq_capacity), in_flight: Slab::with_capacity(2 * sq_capacity), - executing: Vec::with_capacity(options.max_concurrency()), - select_executing: Vec::with_capacity(options.max_concurrency()), + executing: VecDeque::with_capacity(options.max_concurrency()), fs: fs::Filesystem::new(fs_capacity), cq_overflow: options.cq_overflow, cq_dropped: 0, @@ -531,7 +415,7 @@ impl Executor { self.cq_dropped = 0; self.completed().for_each(&mut complete); - while let Some(op) = self.executing.pop() { + while let Some(op) = self.executing.pop_front() { if let Some(in_flight) = self.in_flight.get(op.sqe.key()) && !in_flight.is_active() { @@ -673,7 +557,7 @@ impl Executor { continue; }; if let Some(op) = pending.next() { - self.executing.push(Executing { + self.executing.push_back(Executing { sqe: SqeId(sqe_id), inner: op, }); @@ -687,25 +571,41 @@ impl Executor { fn execute(&mut self, task_selector: &impl TaskSelector, faults: &mut impl FaultInjector) -> bool { let mut progress = false; - self.select_executing.clear(); - self.select_executing.extend( - task_selector - .select_tasks(self.executing.len()) - .into_iter() - .take(self.executing.len()), - ); - self.select_executing.sort_unstable_by(|a, b| b.cmp(a)); - - let mut prev = None; - for i in 0..self.select_executing.len() { - let index = self.select_executing[i]; - assert_ne!(prev, Some(index), "duplicate task selected"); - prev = Some(index); - let op = self.executing.swap_remove(index); - if let Some(delay) = self.execute_op(op, faults) { - self.executing.push(delay); + let mut run = |this: &mut Executor, op| { + if let Some(delay) = this.execute_op(op, faults) { + this.executing.push_back(delay); } progress |= true; + }; + + let Some(remaining) = NonZeroUsize::new(self.executing.len()) else { + return progress; + }; + match task_selector.select_tasks(remaining) { + TaskSelection::Fifo { count } => { + let mut count = count.get(); + while count > 0 { + let Some(op) = self.executing.pop_front() else { + break; + }; + run(self, op); + count -= 1; + } + } + TaskSelection::Any { count, mut select } => { + let mut count = count.get(); + while count > 0 { + let Some(op) = (|| -> Option { + let remaining = NonZeroUsize::new(self.executing.len())?; + let index = select.select_index(remaining); + self.executing.swap_remove_front(index) + })() else { + break; + }; + run(self, op); + count -= 1; + } + } } progress diff --git a/crates/runtime-io/src/sim/faults.rs b/crates/runtime-io/src/sim/faults.rs new file mode 100644 index 00000000000..5428f3ef5f9 --- /dev/null +++ b/crates/runtime-io/src/sim/faults.rs @@ -0,0 +1,198 @@ +use core::{convert::Infallible, num::NonZeroUsize}; + +use crate::sim::{ + executor::{FsyncEffect, InFlight, ReadSector, WriteSector}, + fs::Datasync, + Error, +}; + +/// Interface to inject [Fault]s into the simulator. +/// +/// Submissions are split into one or more operations, or effects, each +/// executing independently and in any order (see [TaskSelector]). For example, +/// a write that spans multiple sectors will yield one atomic write per sector. +/// +/// Before executing an effect, the simulator calls the corresponding `inject_*` +/// method on the fault injector. This allows the implementor to skip, delay or +/// fail the effect. Where applicable, the effect can also be modified: for +/// example, a sector write can be made misdirected by returning a [WriteSector] +/// with a different `sector` from [FaultInjector::inject_write_sector_fault]. +/// +/// A [FaultInjector] can be stateful, which is why the methods take a mutable +/// `self` reference. If desired, effects can be correlated with submissions via +/// the [InFlight] reference passed to the `inject_*` methods. +pub trait FaultInjector { + fn inject_write_sector_fault(&mut self, _: &InFlight, op: WriteSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_read_sector_fault(&mut self, _: &InFlight, op: ReadSector) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_open_fault(&mut self, _: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_create_fault(&mut self, _: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_stat_fault(&mut self, _: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fallocate_fault(&mut self, _: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } + + fn inject_fsync_fault(&mut self, _: &InFlight, op: FsyncEffect) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_fdatasync_fault(&mut self, _: &InFlight, op: Datasync) -> Fault { + Fault::Visible(Effect::Run(op)) + } + + fn inject_noop_fault(&mut self, _: &InFlight) -> Fault<()> { + Fault::Visible(Effect::Run(())) + } +} + +/// The unit [FaultInjector] injects no faults. +impl FaultInjector for () {} + +pub enum Fault { + /// Drop the operation entirely. + /// + /// Note that this is not generally possible in `io-uring`: an SQE always + /// yields a CQE, even if it was cancelled or returned an error. It may be + /// useful occasionally to construct "byzantine" failures. + Skip, + /// Put the operation back onto the queue for later execution. + Delay(T), + /// Execute a visible effect. + Visible(Effect), +} + +impl Fault { + pub(super) fn exec_visible(self, f: impl FnOnce(EitherOrBoth)) -> Option { + match self { + Fault::Skip => None, + Fault::Delay(effect) => Some(effect), + Fault::Visible(visible) => { + visible.exec(f); + None + } + } + } +} + +pub enum Effect { + /// Run the operation as normal. + Run(T), + /// Run the effect, but report an injected error. + RunThenError { effect: T, error: Error }, + /// Skip the effect, but report an injected error. + SkipThenError { error: Error }, +} + +impl Effect { + fn exec(self, f: impl FnOnce(EitherOrBoth)) { + use EitherOrBoth::*; + match self { + Effect::Run(effect) => f(Left(effect)), + Effect::RunThenError { effect, error } => f(Both(effect, error)), + Effect::SkipThenError { error } => f(Right(error)), + } + } +} + +pub(super) enum EitherOrBoth { + Left(T), + Right(U), + Both(T, U), +} + +impl EitherOrBoth { + pub(super) fn traverse(self, f: impl FnOnce(T) -> V, g: impl FnOnce(U) -> V) -> V { + match self { + Self::Left(t) => f(t), + Self::Right(u) => g(u), + Self::Both(t, u) => { + f(t); + g(u) + } + } + } +} + +/// Interface to perturb execution ordering. +/// +/// Submissions are split into one or more operations, or effects, each +/// executing independently. For example, a write that spans multiple sectors +/// will yield one atomic write per sector. +/// +/// Effects could be executed by the kernel in any order, and this trait allows +/// to inject this ordering deterministically (e.g. using a deterministic random +/// number generator). +pub trait TaskSelector { + type IndexSelector<'a>: IndexSelector + where + Self: 'a; + + /// Of `task_count` queued tasks, select the ones to run. + /// + /// Called on each [crate::sim::Executor::tick]. + fn select_tasks(&self, task_count: NonZeroUsize) -> TaskSelection>; +} + +/// Task selection. +pub enum TaskSelection { + /// Run `count` tasks in FIFO order. + Fifo { count: NonZeroUsize }, + /// Select `count` tasks by calling `select` `count` times. + Any { count: NonZeroUsize, select: S }, +} + +/// Select an index in a range. +pub trait IndexSelector { + /// Select a number in the range `0..range_upper`. + fn select_index(&mut self, range_upper: NonZeroUsize) -> usize; +} + +impl IndexSelector for Infallible { + fn select_index(&mut self, _: NonZeroUsize) -> usize { + match *self {} + } +} + +/// [TaskSelector] that runs all queued tasks in FIFO order. +pub struct FifoAll; + +impl TaskSelector for FifoAll { + type IndexSelector<'a> + = Infallible + where + Self: 'a; + + fn select_tasks(&self, count: NonZeroUsize) -> TaskSelection> { + TaskSelection::Fifo { count } + } +} + +/// [TaskSelector] that runs one task at a time, in FIFO order. +pub struct FifoOne; + +impl TaskSelector for FifoOne { + type IndexSelector<'a> + = Infallible + where + Self: 'a; + + fn select_tasks(&self, _: NonZeroUsize) -> TaskSelection> { + TaskSelection::Fifo { + count: NonZeroUsize::MIN, + } + } +} diff --git a/crates/runtime-io/src/sim/mod.rs b/crates/runtime-io/src/sim/mod.rs index 9fad5b48f2a..e7d63662771 100644 --- a/crates/runtime-io/src/sim/mod.rs +++ b/crates/runtime-io/src/sim/mod.rs @@ -16,11 +16,14 @@ use completion::CompletionHandle; mod executor; use executor::{Executor, Sqe}; +mod faults; +pub use faults::{FaultInjector, FifoAll, FifoOne, IndexSelector, TaskSelector}; + mod fs; pub use fs::File; pub use crate::{ - sim::executor::{FaultInjector, LinkKind, Options, TaskSelector}, + sim::executor::{LinkKind, Options}, SECTOR_SIZE, }; @@ -261,15 +264,39 @@ impl SpacetimeIO for SimulatorIO { #[cfg(test)] mod tests { - use spacetimedb_runtime_core::sim::Rng; + use core::num::NonZeroUsize; - use crate::SECTOR_SIZE64; + use spacetimedb_runtime_core::sim::Rng; use super::*; + use crate::{sim::faults::TaskSelection, SECTOR_SIZE64}; + + struct RandomTaskSelector<'a> { + rng: &'a Rng, + } + + impl TaskSelector for RandomTaskSelector<'_> { + type IndexSelector<'a> + = &'a Rng + where + Self: 'a; + + fn select_tasks(&self, task_count: NonZeroUsize) -> TaskSelection> { + let count = NonZeroUsize::new(self.rng.index(task_count.get())).unwrap_or(NonZeroUsize::MIN); + if self.rng.sample_probability(0.5) { + TaskSelection::Fifo { count } + } else { + TaskSelection::Any { + count, + select: self.rng, + } + } + } + } - impl TaskSelector for Rng { - fn select_tasks(&self, task_count: usize) -> impl IntoIterator { - (task_count > 0).then(|| self.index(task_count)) + impl IndexSelector for &Rng { + fn select_index(&mut self, range_upper: NonZeroUsize) -> usize { + self.index(range_upper.get()) } } @@ -292,7 +319,7 @@ mod tests { fn run(&self, f: impl FnOnce(&SimulatorIO) -> Completion) -> T { let fut = self.rt.spawn_local(f(&self.io)); - while self.io.tick(&self.rng, &mut ()) {} + while self.io.tick(&RandomTaskSelector { rng: &self.rng }, &mut ()) {} self.rt.block_on(fut).unwrap() }