diff --git a/Cargo.lock b/Cargo.lock index bb51866f..2905081d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1480,6 +1480,7 @@ dependencies = [ "kiln-host", "kiln-platform", "once_cell", + "serial_test", "tract-onnx", ] diff --git a/kiln-runtime/src/stackless/engine.rs b/kiln-runtime/src/stackless/engine.rs index 93443055..5599907b 100644 --- a/kiln-runtime/src/stackless/engine.rs +++ b/kiln-runtime/src/stackless/engine.rs @@ -729,26 +729,35 @@ impl StacklessEngine { let sig = get_wasi_function_signature(interface, function); let needs_retptr = sig.as_ref().map_or(false, |s| s.result_needs_retptr()); - // ON-DEMAND ALLOCATION for input-stream reads (SR-38): the returned list is - // owned and freed by the guest, so a fresh cabi_realloc'd buffer is allocated per - // read (a reused/fixed buffer would be freed by the guest → allocator corruption). - // Done before borrowing the handler so the &mut self call has no conflicting borrow. - let read_alloc: Option<(u32, u32)> = if interface.contains("wasi:io/streams") - && (function.contains("input-stream.read") - || function.contains("input-stream.blocking-read")) - { - let len = args.get(1).and_then(|v| match v { - Value::I64(n) => Some(*n as u32), - Value::I32(n) => Some(*n as u32), - _ => None, - }).unwrap_or(0); - if len > 0 { - self.allocate_wasi_read_buffer(instance_id, len).ok().flatten() + // ON-DEMAND ALLOCATION for guest-owned list/string returns (SR-38, SR-39): the + // returned list/string is owned and freed by the guest, so a fresh cabi_realloc'd + // buffer is allocated per call (a reused/fixed buffer would be freed by the guest → + // allocator corruption). Done before borrowing the handler so the &mut self call has + // no conflicting borrow. + let read_alloc: Option<(u32, u32)> = { + let want = if interface.contains("wasi:io/streams") + && (function.contains("input-stream.read") + || function.contains("input-stream.blocking-read")) + { + // input-stream read: size to the requested len (args = [self, len, ...]). + args.get(1).and_then(|v| match v { + Value::I64(n) => Some(*n as u32), + Value::I32(n) => Some(*n as u32), + _ => None, + }).unwrap_or(0) + } else if interface.contains("wasi:filesystem/types") + && function.contains("read-directory-entry") + { + // directory-entry name string: no len arg, use a filename-sized buffer. + 512 + } else { + 0 + }; + if want > 0 { + self.allocate_wasi_read_buffer(instance_id, want).ok().flatten() } else { None } - } else { - None }; let instance = self.instances.get(&instance_id) diff --git a/kiln-runtime/src/wasip2_host.rs b/kiln-runtime/src/wasip2_host.rs index 2bca4fcb..a1b7dc87 100644 --- a/kiln-runtime/src/wasip2_host.rs +++ b/kiln-runtime/src/wasip2_host.rs @@ -434,6 +434,26 @@ pub fn get_wasi_function_signature(interface: &str, function: &str) -> Option Some(WasiFunctionSignature::new( + vec![WasiComponentType::Handle], + vec![WasiComponentType::Result( + Some(Box::new(WasiComponentType::Handle)), // directory-entry-stream + Some(Box::new(WasiComponentType::U8)), // error-code + )], + )), + ("wasi:filesystem/types", "[method]directory-entry-stream.read-directory-entry") => Some(WasiFunctionSignature::new( + vec![WasiComponentType::Handle], + vec![WasiComponentType::Result( + // option where directory-entry = record { type: descriptor-type, name: string }. + // The enum descriptor-type lowers as a u8 (padded to 4) — modelled as U8; the record as a Tuple. + Some(Box::new(WasiComponentType::Option(Box::new(WasiComponentType::Tuple(vec![ + WasiComponentType::U8, + WasiComponentType::String, + ]))))), + Some(Box::new(WasiComponentType::U8)), // error-code + )], + )), ("wasi:filesystem/types", "[resource-drop]descriptor") | ("wasi:filesystem/types", "[resource-drop]directory-entry-stream") => Some(WasiFunctionSignature::new( vec![WasiComponentType::Handle], diff --git a/kiln-wasi/Cargo.toml b/kiln-wasi/Cargo.toml index 086b6320..6880b8f9 100644 --- a/kiln-wasi/Cargo.toml +++ b/kiln-wasi/Cargo.toml @@ -79,6 +79,12 @@ safety-critical = ["asil-b"] # Maximum safety level for WASI # Development and testing features # (Add more features as needed) +[dev-dependencies] +# Dispatcher tests build a WasiDispatcher, which draws on the global capability +# memory budget; run them serially to avoid cross-test budget contention +# (see CLAUDE.md — CFI/engine tests use serial_test). +serial_test = "3.4" + [lints.rust] unexpected_cfgs = { level = "allow", check-cfg = ['cfg(test)'] } missing_docs = "deny" diff --git a/kiln-wasi/src/dispatcher.rs b/kiln-wasi/src/dispatcher.rs index 9610d625..0ee5b1df 100644 --- a/kiln-wasi/src/dispatcher.rs +++ b/kiln-wasi/src/dispatcher.rs @@ -234,6 +234,11 @@ pub struct WasiDispatcher { /// terminate at EOF (return stream-error::closed) instead of re-reading from 0. SR-38. #[cfg(feature = "std")] stream_offsets: HashMap, + /// Open directory-entry-streams: a snapshot of (descriptor-type, name) entries plus a + /// cursor, keyed by the stream handle from read-directory. read-directory-entry walks + /// the cursor and returns none at the end. SR-39. + #[cfg(feature = "std")] + dir_streams: HashMap, usize)>, } impl WasiDispatcher { @@ -282,6 +287,8 @@ impl WasiDispatcher { read_buf_alloc: None, #[cfg(feature = "std")] stream_offsets: HashMap::new(), + #[cfg(feature = "std")] + dir_streams: HashMap::new(), }) } @@ -2273,6 +2280,89 @@ impl WasiDispatcher { Ok(vec![CoreValue::I32(0), CoreValue::I32(stream_handle as i32)]) } + #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + ("wasi:filesystem/types", "[method]descriptor.read-directory") => { + // read-directory(self) -> result + // Snapshot the directory's entries now; the guest walks them via + // read-directory-entry. `.`/`..` are excluded (std::fs::read_dir omits them), + // matching preview2 semantics. SR-39. + let fd = match args.first() { + Some(CoreValue::I32(v)) => *v as u32, + _ => return Err(Error::wasi_invalid_argument("read-directory: missing descriptor")), + }; + let dir_path = self.fd_table.get(&fd).and_then(|e| match &e.fd_type { + FileDescriptorType::PreopenDirectory(p) | FileDescriptorType::RegularFile(p) => Some(p.clone()), + _ => None, + }); + let dir_path = match dir_path { + Some(p) => p, + None => return Ok(vec![CoreValue::I32(1), CoreValue::I32(8)]), // err::bad-descriptor + }; + let mut entries: Vec<(u8, String)> = Vec::new(); + match std::fs::read_dir(&dir_path) { + Ok(rd) => { + for ent in rd.flatten() { + // descriptor-type enum: unknown=0, block-device=1, character-device=2, + // directory=3, fifo=4, symbolic-link=5, regular-file=6, socket=7. + let ty = ent.file_type().ok(); + let tycode: u8 = match ty { + Some(t) if t.is_dir() => 3, + Some(t) if t.is_symlink() => 5, + Some(t) if t.is_file() => 6, + _ => 0, + }; + entries.push((tycode, ent.file_name().to_string_lossy().into_owned())); + } + } + Err(_) => return Ok(vec![CoreValue::I32(1), CoreValue::I32(8)]), + } + let handle = self.resource_manager.create_input_stream(&format!("dir:{}", fd))?; + self.dir_streams.insert(handle, (entries, 0)); + Ok(vec![CoreValue::I32(0), CoreValue::I32(handle as i32)]) + } + + #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + ("wasi:filesystem/types", "[method]directory-entry-stream.read-directory-entry") => { + // read-directory-entry(self) -> result, error-code> + // directory-entry = record { type: descriptor-type, name: string }. Core values: + // ok(some): [I32(0), I32(1), I32(type), I32(name_ptr), I32(name_len)] + // ok(none): [I32(0), I32(0)] + // The name string goes in a fresh cabi_realloc'd buffer (read_buf_alloc). SR-39. + let stream_handle = match args.first() { + Some(CoreValue::I32(v)) => *v as u32, + _ => return Err(Error::wasi_invalid_argument("read-directory-entry: missing stream")), + }; + let read_buf = self.read_buf_alloc.take(); + let next = self.dir_streams.get(&stream_handle).and_then(|(entries, cursor)| { + entries.get(*cursor).cloned() + }); + match next { + None => Ok(vec![CoreValue::I32(0), CoreValue::I32(0)]), // ok(none) + Some((tycode, name)) => { + let (buf_ptr, buf_size) = match read_buf { + Some((p, s)) => (p, s as usize), + None => return Err(Error::runtime_error( + "read-directory-entry requires a cabi_realloc'd buffer (read_buf_alloc not set)", + )), + }; + let mem = memory.ok_or_else(|| Error::wasi_invalid_argument("read-directory-entry: no memory"))?; + let name_bytes = name.as_bytes(); + let n = core::cmp::min(name_bytes.len(), buf_size); + mem.write_bytes(buf_ptr, &name_bytes[..n])?; + if let Some((_, cursor)) = self.dir_streams.get_mut(&stream_handle) { + *cursor += 1; + } + Ok(vec![ + CoreValue::I32(0), // result::ok + CoreValue::I32(1), // option::some + CoreValue::I32(tycode as i32), // descriptor-type + CoreValue::I32(buf_ptr as i32), // name ptr + CoreValue::I32(n as i32), // name len + ]) + } + } + } + #[cfg(all(feature = "wasi-filesystem", feature = "std"))] ("wasi:filesystem/types", "[method]descriptor.metadata-hash") | ("wasi:filesystem/types", "[method]descriptor.metadata-hash-at") => { @@ -2765,6 +2855,7 @@ mod tests { /// usable preopen instead of a list backed by memory it never allocated. // rivet: verifies SR-37 #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + #[serial_test::serial] #[test] fn get_directories_uses_cabi_realloc_allocation_not_retptr_scribble() -> Result<()> { use kiln_foundation::traits::SliceMemory; @@ -2823,6 +2914,7 @@ mod tests { /// unowned guest memory (the pre-fix behavior that produced ENOENT). // rivet: verifies SR-37 #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + #[serial_test::serial] #[test] fn get_directories_fails_loud_without_allocation() -> Result<()> { use kiln_foundation::traits::SliceMemory; @@ -2848,6 +2940,7 @@ mod tests { /// handed a fixed unowned buffer (the pre-fix bug that corrupted the guest). // rivet: verifies SR-38 #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + #[serial_test::serial] #[test] fn blocking_read_returns_bytes_in_owned_buffer_and_eofs() -> Result<()> { use kiln_foundation::traits::SliceMemory; @@ -2915,6 +3008,7 @@ mod tests { /// (the pre-fix behavior that corrupted the guest allocator → E07DA). // rivet: verifies SR-38 #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + #[serial_test::serial] #[test] fn blocking_read_fails_loud_without_buffer() -> Result<()> { use kiln_foundation::traits::SliceMemory; @@ -2945,6 +3039,85 @@ mod tests { Ok(()) } + /// SR-39 / #405: read-directory snapshots a directory's entries and + /// read-directory-entry walks them, returning each name in a cabi_realloc'd + /// buffer and ok(none) at the end — so a component's std::fs::read_dir works. + // rivet: verifies SR-39 + #[cfg(all(feature = "wasi-filesystem", feature = "std"))] + #[serial_test::serial] + #[test] + fn read_directory_enumerates_entries_then_none() -> Result<()> { + use kiln_foundation::traits::SliceMemory; + use kiln_foundation::HostImportHandler; + use std::io::Write as _; + MemoryInitializer::ensure_initialized()?; + let mut dispatcher = WasiDispatcher::with_defaults()?; + + // Temp directory with two known files. + let dir = std::env::temp_dir().join("kiln_sr39_dir"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::File::create(dir.join("a.txt")).unwrap().write_all(b"a").unwrap(); + std::fs::File::create(dir.join("b.txt")).unwrap().write_all(b"b").unwrap(); + + // Register the directory as an open descriptor. + let dir_fd = 9u32; + dispatcher.fd_table.insert(dir_fd, FileDescriptorEntry { + fd_type: FileDescriptorType::PreopenDirectory(dir.clone()), + read: true, + write: false, + }); + + let mem = SliceMemory::with_size(0x10000); + + // read-directory → ok(stream handle). + let rd = dispatcher.dispatch_core( + "wasi:filesystem/types", + "[method]descriptor.read-directory", + &[kiln_foundation::values::Value::I32(dir_fd as i32)], + Some(&mem), + )?; + assert!(matches!(rd[0], kiln_foundation::values::Value::I32(0)), "read-directory ok"); + let stream = match rd[1] { kiln_foundation::values::Value::I32(h) => h as u32, _ => panic!("no handle") }; + + // Walk entries; each name lands in a fresh buffer. + let mut names = Vec::new(); + let name_ptr = 0x6000u32; + for _ in 0..2 { + dispatcher.set_read_buffer_allocation(name_ptr, 256); + let e = dispatcher.dispatch_core( + "wasi:filesystem/types", + "[method]directory-entry-stream.read-directory-entry", + &[kiln_foundation::values::Value::I32(stream as i32)], + Some(&mem), + )?; + assert_eq!(e.len(), 5, "ok(some(record)) lowers to [ok, some, type, ptr, len]"); + assert!(matches!(e[0], kiln_foundation::values::Value::I32(0))); + assert!(matches!(e[1], kiln_foundation::values::Value::I32(1))); + assert!(matches!(e[2], kiln_foundation::values::Value::I32(6)), "regular-file = 6"); + let len = match e[4] { kiln_foundation::values::Value::I32(l) => l as usize, _ => 0 }; + let mut buf = vec![0u8; len]; + mem.read_bytes(name_ptr, &mut buf)?; + names.push(String::from_utf8(buf).unwrap()); + } + names.sort(); + assert_eq!(names, vec!["a.txt".to_string(), "b.txt".to_string()]); + + // Next call → ok(none). + let end = dispatcher.dispatch_core( + "wasi:filesystem/types", + "[method]directory-entry-stream.read-directory-entry", + &[kiln_foundation::values::Value::I32(stream as i32)], + Some(&mem), + )?; + assert_eq!(end.len(), 2, "ok(none) lowers to [ok, none]"); + assert!(matches!(end[0], kiln_foundation::values::Value::I32(0))); + assert!(matches!(end[1], kiln_foundation::values::Value::I32(0)), "none"); + + let _ = std::fs::remove_dir_all(&dir); + Ok(()) + } + #[test] fn test_dispatcher_creation() -> Result<()> { MemoryInitializer::ensure_initialized()?; diff --git a/safety/requirements/functional-requirements.yaml b/safety/requirements/functional-requirements.yaml index a49e49c6..4b19c138 100644 --- a/safety/requirements/functional-requirements.yaml +++ b/safety/requirements/functional-requirements.yaml @@ -1032,7 +1032,7 @@ artifacts: - id: SR-39 type: requirement title: Component directory enumeration (read-directory / read-directory-entry) works under --wasi-fs - status: proposed + status: verified description: "The last #405 blocker after SR-37 (get-directories) and SR-38 (file read) landed: with both fixed, a preview2 component reads a file end-to-end (READ:CANARY matches wasmtime), but std::fs::read_dir('.') still fails. Measured trace: open-at(dir, O_DIRECTORY)→descriptor 5 → wasi:filesystem/types::[method]descriptor.read-directory(5, retptr) → [Runtime][E07DA] Function not found in exports. Root cause: read-directory and directory-entry-stream.read-directory-entry are UNIMPLEMENTED — dispatch_core has only a stale 'readdir' arm (different name) + resource-drop, and wasip2_host.rs has NO get_wasi_function_signature entry for either, so the calls fall through. Needed: (a) descriptor.read-directory(self) -> result — create a stream resource, snapshot std::fs::read_dir entries + a cursor keyed by the new handle; (b) directory-entry-stream.read-directory-entry(self) -> result, error-code> where directory-entry = record { type: descriptor-type (enum), name: string } — return the next entry (name string in a cabi_realloc'd guest buffer, reusing the SR-38 on-demand read-buffer mechanism / a new one) or none at end; (c) signatures in wasip2_host.rs; (d) verify lower_type_to_memory handles Record + Enum + Option (add if missing). Differential oracle: wasmtime run --dir .::. prints DIR:[inside.txt]. Closes #405 end-to-end; cut v0.4.1 when done." tags: [kilnd, wasi, filesystem, component, canonical-abi, directory, bug] links: @@ -1045,3 +1045,15 @@ artifacts: model: claude-opus-4-8 timestamp: 2026-07-10T00:00:00Z release: v0.4.1 + + - id: SR-40 + type: requirement + title: kiln-wasi test suite is deterministic under parallel execution (global capability-budget contention) + status: proposed + description: "Bit-rot: the kiln-wasi test suite is intermittently RED under parallel `cargo test` — a different subset of capability/allocation tests fails each run (observed: test_environment_var_management, test_capability_presets, test_minimal_capabilities, test_capability_defaults_by_safety_level, test_safety_level_override, test_safety_aware_allocation_enforcement). Confirmed PRE-EXISTING on clean main (fails ~1/3 runs with zero local changes), so CI is currently green only by luck/retry. Root cause: these tests each draw from the shared global capability memory budget (via MemoryInitializer + WasiCapabilities/WasiDispatcher construction) without serial marks, so concurrent execution exhausts the budget and allocations fail. ~19 budget-drawing call sites across kiln-wasi/src/{capabilities.rs,dispatcher.rs,lib.rs,host_provider/*} + tests/. Fix options: mark all budget-drawing tests #[serial_test::serial] (they share one lock → never concurrent), or raise/scope the test-time budget, or run the crate's tests single-threaded. SR-37/38/39 already added serial_test as a kiln-wasi dev-dep and marked the 5 new dispatcher tests; this requirement is to finish the job for the pre-existing tests. Oracle: `cargo test -p kiln-wasi --features wasi-filesystem` green 10/10 consecutive runs." + tags: [kiln-wasi, tests, flaky, bit-rot, ci] + provenance: + created-by: ai + model: claude-opus-4-8 + timestamp: 2026-07-10T00:10:00Z + release: v0.5.0