diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index c5946db1d..6f6d2236d 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -61,7 +61,7 @@ wasmer-derive = { path = "../derive", version = "=7.1.0" } wasmer-types = { path = "../types", version = "=7.1.0" } target-lexicon = { workspace = true, default-features = false } # - Optional dependencies for `sys`. -wasmer-compiler-singlepass = { path = "../compiler-singlepass", version = "=7.1.0", optional = true } +wasmer-compiler-singlepass = { path = "../compiler-singlepass", version = "=7.1.0", optional = true, default-features = false, features = ["std", "unwind"] } wasmer-compiler-cranelift = { path = "../compiler-cranelift", version = "=7.1.0", optional = true } wasmer-compiler-llvm = { path = "../compiler-llvm", version = "=7.1.0", optional = true } corosensei = { workspace = true, optional = true } @@ -145,6 +145,7 @@ compiler = [ "wasmer-compiler/compiler", ] singlepass = ["compiler", "wasmer-compiler-singlepass"] +singlepass_rayon = ["wasmer-compiler-singlepass?/rayon"] cranelift = ["compiler", "wasmer-compiler-cranelift"] llvm = ["compiler", "wasmer-compiler-llvm"] diff --git a/lib/api/src/backend/sys/entities/module.rs b/lib/api/src/backend/sys/entities/module.rs index 110affcfc..c56cd1547 100644 --- a/lib/api/src/backend/sys/entities/module.rs +++ b/lib/api/src/backend/sys/entities/module.rs @@ -219,6 +219,17 @@ impl Module { } } + #[cfg(not(target_arch = "wasm32"))] + /// Extracts all local function locations for external usage + /// 3 parameters are: local function index, starting address, function length + pub fn local_function_infos(&self) -> Vec<(u32, usize, usize)> { + self.artifact + .finished_function_extents() + .into_iter() + .map(|(index, extent)| (index.as_u32(), extent.ptr.0 as usize, extent.length)) + .collect() + } + pub(crate) fn name(&self) -> Option<&str> { self.info().name.as_deref() } diff --git a/lib/api/src/entities/store/context.rs b/lib/api/src/entities/store/context.rs index 3c4285bf3..dd3a9d6f6 100644 --- a/lib/api/src/entities/store/context.rs +++ b/lib/api/src/entities/store/context.rs @@ -149,6 +149,42 @@ impl StoreContext { }) } + /// # Safety + /// + /// Only use this in cases where you know some other code is in process + /// of StorePtrWrapper (and can trigger a drop). One example is using + /// wasmer function in coroutines which can yield. + pub(crate) unsafe fn force_clean(id: StoreId) { + if !Self::is_active(id) { + return; + } + STORE_CONTEXT_STACK.with(|cell| { + let mut stack = cell.borrow_mut(); + let top = stack.pop().expect("An entry must exist!"); + assert_eq!(top.id, id); + }); + } + + /// # Safety + /// + /// Only use this in cases where you know some other code is in process + /// of StorePtrWrapper (and can trigger a drop). One example is using + /// wasmer function in coroutines which can yield. + pub(crate) unsafe fn force_create(store_ptr: *mut StoreInner) { + let store_id = unsafe { store_ptr.as_ref().unwrap().objects.id() }; + if Self::is_active(store_id) { + return; + } + STORE_CONTEXT_STACK.with(|cell| { + let mut stack = cell.borrow_mut(); + stack.push(Self { + id: store_id, + borrow_count: 1, + entry: UnsafeCell::new(StoreContextEntry::Sync(store_ptr)), + }); + }); + } + /// Returns true if there are no active store context entries. pub(crate) fn is_empty() -> bool { STORE_CONTEXT_STACK.with(|cell| { @@ -346,6 +382,7 @@ impl Drop for StorePtrWrapper { let id = self.as_mut().objects_mut().id(); STORE_CONTEXT_STACK.with(|cell| { let mut stack = cell.borrow_mut(); + let ids: Vec<_> = stack.iter().map(|s| (s.id, s.borrow_count)).collect(); let top = stack .last_mut() .expect("No store context installed on this thread"); diff --git a/lib/api/src/entities/store/mod.rs b/lib/api/src/entities/store/mod.rs index e1aee0709..ee992fb94 100644 --- a/lib/api/src/entities/store/mod.rs +++ b/lib/api/src/entities/store/mod.rs @@ -120,6 +120,26 @@ impl Store { self.inner.store.engine_mut() } + #[cfg(feature = "sys")] + #[inline] + /// Cleaning current store context, see StoreContext::force_clean + /// for details + pub fn force_clean(&mut self) { + unsafe { + StoreContext::force_clean(self.id()); + } + } + + #[cfg(feature = "sys")] + #[inline] + /// Re-create current store context, see StoreContext::force_create + /// for details + pub fn force_create(&mut self) { + unsafe { + StoreContext::force_create(self.as_store_mut().inner as *mut _); + } + } + /// Checks whether two stores are identical. A store is considered /// equal to another store if both have the same engine. pub fn same(a: &Self, b: &Self) -> bool { diff --git a/lib/compiler-llvm/src/config.rs b/lib/compiler-llvm/src/config.rs index c28f36760..6504d1dd9 100644 --- a/lib/compiler-llvm/src/config.rs +++ b/lib/compiler-llvm/src/config.rs @@ -321,7 +321,7 @@ impl LLVM { let target_triple = self.target_triple(target); let llvm_target = InkwellTarget::from_triple(&target_triple).unwrap(); - let mut llvm_target_machine_options = TargetMachineOptions::new() + let llvm_target_machine_options = TargetMachineOptions::new() .set_cpu(match triple.architecture { Architecture::Riscv64(_) => "generic-rv64", Architecture::Riscv32(_) => "generic-rv32", @@ -329,8 +329,10 @@ impl LLVM { _ => "generic", }) .set_features(match triple.architecture { - Architecture::Riscv64(_) => "+m,+a,+c,+d,+f", - Architecture::Riscv32(_) => "+m,+a,+c,+d,+f", + // Our RISC-V experiment only uses rv[32|64]im. Floating point, compressed + // instruction, atomic operations are not supported. + Architecture::Riscv64(_) => "+m", + Architecture::Riscv32(_) => "+m", Architecture::LoongArch64 => "+f,+d", _ => &llvm_cpu_features, }) @@ -346,9 +348,9 @@ impl LLVM { } _ => self.code_model(self.target_binary_format(target)), }); - if let Architecture::Riscv64(_) = triple.architecture { - llvm_target_machine_options = llvm_target_machine_options.set_abi("lp64d"); - } + // if let Architecture::Riscv64(_) = triple.architecture { + // llvm_target_machine_options = llvm_target_machine_options.set_abi("lp64d"); + // } let target_machine = llvm_target .create_target_machine_from_options(&target_triple, llvm_target_machine_options) .unwrap(); diff --git a/lib/compiler-llvm/src/object_file.rs b/lib/compiler-llvm/src/object_file.rs index 7265bca86..1668bf360 100644 --- a/lib/compiler-llvm/src/object_file.rs +++ b/lib/compiler-llvm/src/object_file.rs @@ -90,6 +90,64 @@ static LIBCALLS_ELF: phf::Map<&'static str, LibCall> = phf::phf_map! { "wasmer_eh_personality2" => LibCall::EHPersonality2, "wasmer_vm_dbg_usize" => LibCall::DebugUsize, "wasmer_vm_dbg_str" => LibCall::DebugStr, + // Floating-point & 64-bit mul/div routie functions. Those might + // be generated by LLVM when generating objects for 32-bit platforms, + // or platforms without floating point supports. Modern Rust & C toolchains + // provide implementations for thos functions by default. + "__adddf3" => LibCall::Adddf3, + "__addsf3" => LibCall::Addsf3, + "__divdf3" => LibCall::Divdf3, + "__divdi3" => LibCall::Divdi3, + "__divsf3" => LibCall::Divsf3, + "__divsi3" => LibCall::Divsi3, + "__eqdf2" => LibCall::Eqdf2, + "__eqsf2" => LibCall::Eqsf2, + "__extendsfdf2" => LibCall::Extendsfdf2, + "__fixdfdi" => LibCall::Fixdfdi, + "__fixdfsi" => LibCall::Fixdfsi, + "__fixsfdi" => LibCall::Fixsfdi, + "__fixsfsi" => LibCall::Fixsfsi, + "__fixunsdfdi" => LibCall::Fixunsdfdi, + "__fixunsdfsi" => LibCall::Fixunsdfsi, + "__fixunssfdi" => LibCall::Fixunssfdi, + "__fixunssfsi" => LibCall::Fixunssfsi, + "__floatdidf" => LibCall::Floatdidf, + "__floatdisf" => LibCall::Floatdisf, + "__floatsidf" => LibCall::Floatsidf, + "__floatsisf" => LibCall::Floatsisf, + "__floatundidf" => LibCall::Floatundidf, + "__floatundisf" => LibCall::Floatundisf, + "__floatunsidf" => LibCall::Floatunsidf, + "__floatunsisf" => LibCall::Floatunsisf, + "__gedf2" => LibCall::Gedf2, + "__gesf2" => LibCall::Gesf2, + "__gtdf2" => LibCall::Gtdf2, + "__gtsf2" => LibCall::Gtsf2, + "__ledf2" => LibCall::Ledf2, + "__lesf2" => LibCall::Lesf2, + "__ltdf2" => LibCall::Ltdf2, + "__ltsf2" => LibCall::Ltsf2, + "__moddi3" => LibCall::Moddi3, + "__modsi3" => LibCall::Modsi3, + "__muldf3" => LibCall::Muldf3, + "__muldi3" => LibCall::Muldi3, + "__mulsf3" => LibCall::Mulsf3, + "__mulsi3" => LibCall::Mulsi3, + "__nedf2" => LibCall::Nedf2, + "__negdf2" => LibCall::Negdf2, + "__negsf2" => LibCall::Negsf2, + "__nesf2" => LibCall::Nesf2, + "__subdf3" => LibCall::Subdf3, + "__subsf3" => LibCall::Subsf3, + "__truncdfsf2" => LibCall::Truncdfsf2, + "__udivdi3" => LibCall::Udivdi3, + "__udivsi3" => LibCall::Udivsi3, + "__umoddi3" => LibCall::Umoddi3, + "__umodsi3" => LibCall::Umodsi3, + "__unorddf2" => LibCall::Unorddf2, + "__unordsf2" => LibCall::Unordsf2, + "memset" => LibCall::Memset, + "sqrt" => LibCall::Sqrt, }; static LIBCALLS_MACHO: phf::Map<&'static str, LibCall> = phf::phf_map! { @@ -755,7 +813,7 @@ where object::macho::ARM64_RELOC_ADDEND => RelocationKind::MachoArm64RelocAddend, _ => { return Err(CompileError::Codegen(format!( - "unknown relocation {reloc:?}", + "unknown relocation 1 {reloc:?}", ))); } }, @@ -780,13 +838,14 @@ where object::macho::X86_64_RELOC_TLV => RelocationKind::MachoX86_64RelocTlv, _ => { return Err(CompileError::Codegen(format!( - "unknown relocation {reloc:?}" + "unknown relocation 2 {reloc:?}" ))); } }, _ => { return Err(CompileError::Codegen(format!( - "unknown relocation {reloc:?}", + "unknown relocation 3 {reloc:?}, arch: {:?}", + obj.architecture(), ))); } }; diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index d66812709..7282e00c1 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -721,24 +721,7 @@ impl Artifact { return Ok(()); // already done } - let finished_function_extents = self - .allocated - .as_ref() - .expect("It must be allocated") - .finished_functions - .values() - .copied() - .zip( - self.allocated - .as_ref() - .expect("It must be allocated") - .finished_function_lengths - .values() - .copied(), - ) - .map(|(ptr, length)| FunctionExtent { ptr, length }) - .collect::>() - .into_boxed_slice(); + let finished_function_extents = self.finished_function_extents(); let frame_info_registration = &mut self .allocated @@ -785,6 +768,27 @@ impl Artifact { .finished_functions } + /// Returns function extents for external processing, e.g., debugging & profiling + pub fn finished_function_extents(&self) -> BoxedSlice { + self.allocated + .as_ref() + .expect("It must be allocated") + .finished_functions + .values() + .copied() + .zip( + self.allocated + .as_ref() + .expect("It must be allocated") + .finished_function_lengths + .values() + .copied(), + ) + .map(|(ptr, length)| FunctionExtent { ptr, length }) + .collect::>() + .into_boxed_slice() + } + /// Returns the function call trampolines allocated in memory of this /// `Artifact`, ready to be run. pub fn finished_function_call_trampolines(&self) -> &BoxedSlice { diff --git a/lib/compiler/src/engine/link.rs b/lib/compiler/src/engine/link.rs index e25ffc737..987486a62 100644 --- a/lib/compiler/src/engine/link.rs +++ b/lib/compiler/src/engine/link.rs @@ -213,9 +213,9 @@ fn apply_relocation( RelocationKind::Aarch64AdrPrelPgHi21 => unsafe { let (reloc_address, delta) = r.for_address(body, target_func_address as u64); - let delta = delta as isize; + let delta = delta as i64; assert!( - ((-1 << 32)..(1 << 32)).contains(&delta), + ((-1i64 << 32)..(1i64 << 32)).contains(&delta), "can't generate page-relative relocation with ±4GB `adrp` instruction" ); diff --git a/lib/compiler/src/engine/unwind/dummy.rs b/lib/compiler/src/engine/unwind/dummy.rs index 0a34e29fd..89bacbe54 100644 --- a/lib/compiler/src/engine/unwind/dummy.rs +++ b/lib/compiler/src/engine/unwind/dummy.rs @@ -1,6 +1,6 @@ //! Module for Dummy unwind registry. -use wasmer_types::CompiledFunctionUnwindInfo; +use crate::types::unwind::CompiledFunctionUnwindInfoReference; /// Represents a registry of function unwind information when the host system /// support any one in specific. @@ -18,14 +18,28 @@ impl DummyUnwindRegistry { _base_address: usize, _func_start: u32, _func_len: u32, - _info: &CompiledFunctionUnwindInfo, + _info: &CompiledFunctionUnwindInfoReference, + ) -> Result<(), String> { + // Do nothing + Ok(()) + } + + pub(crate) fn register_compact_unwind( + &mut self, + _compact_unwind: Option<&[u8]>, + _eh_personality_addr_in_got: Option, ) -> Result<(), String> { // Do nothing Ok(()) } /// Publishes all registered functions. - pub fn publish(&mut self, eh_frame: Option<&[u8]>) -> Result<(), String> { + pub fn publish(&mut self, _eh_frame: Option<&[u8]>) -> Result<(), String> { + // Do nothing + Ok(()) + } + + pub fn publish_eh_frame(&mut self, _eh_frame: Option<&[u8]>) -> Result<(), String> { // Do nothing Ok(()) } diff --git a/lib/compiler/src/engine/unwind/systemv.rs b/lib/compiler/src/engine/unwind/systemv.rs index 11ef2ad9e..e604ce06d 100644 --- a/lib/compiler/src/engine/unwind/systemv.rs +++ b/lib/compiler/src/engine/unwind/systemv.rs @@ -289,4 +289,4 @@ impl Drop for UnwindRegistry { } } } -} +} \ No newline at end of file diff --git a/lib/compiler/src/object/module.rs b/lib/compiler/src/object/module.rs index e38c70ccf..9ecb2dad8 100644 --- a/lib/compiler/src/object/module.rs +++ b/lib/compiler/src/object/module.rs @@ -9,7 +9,7 @@ use crate::{ }, }; use object::{ - FileFlags, RelocationEncoding, RelocationFlags, RelocationKind, SectionKind, SymbolFlags, + RelocationEncoding, RelocationFlags, RelocationKind, SectionKind, SymbolFlags, SymbolKind, SymbolScope, elf, macho, write::{ Object, Relocation, StandardSection, StandardSegment, Symbol as ObjSymbol, SymbolId, @@ -67,15 +67,15 @@ pub fn get_object_for_target(triple: &Triple) -> Result, ObjectE Endianness::Big => object::Endianness::Big, }; - let mut object = Object::new(obj_binary_format, obj_architecture, obj_endianness); + let object = Object::new(obj_binary_format, obj_architecture, obj_endianness); - if let Architecture::Riscv64(_) = triple.architecture { - object.flags = FileFlags::Elf { - e_flags: elf::EF_RISCV_FLOAT_ABI_DOUBLE, - os_abi: 2, - abi_version: 0, - }; - } + // if let Architecture::Riscv64(_) = triple.architecture { + // object.flags = FileFlags::Elf { + // e_flags: elf::EF_RISCV_FLOAT_ABI_DOUBLE, + // os_abi: 2, + // abi_version: 0, + // }; + // } Ok(object) } diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 83329b9a9..56ce84f09 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -30,7 +30,7 @@ wasmparser = { workspace = true, default-features = false, optional = true } # `rand` uses `getrandom` transitively, and to be able to # compile the project for `js`, we need to enable this feature -[dependencies.getrandom] +[target.'cfg(not(target_os = "zkvm"))'.dependencies.getrandom] workspace = true features = ["wasm_js"] diff --git a/lib/types/src/libcalls.rs b/lib/types/src/libcalls.rs index 7aaada05a..f529717d6 100644 --- a/lib/types/src/libcalls.rs +++ b/lib/types/src/libcalls.rs @@ -146,6 +146,168 @@ pub enum LibCall { DebugUsize, /// debug_str DebugStr, + + /// __adddf3 + Adddf3, + + /// __addsf3 + Addsf3, + + /// __divdf3 + Divdf3, + + /// __divdi3 + Divdi3, + + /// __divsf3 + Divsf3, + + /// __divsi3 + Divsi3, + + /// __eqdf2 + Eqdf2, + + /// __eqsf2 + Eqsf2, + + /// __extendsfdf2 + Extendsfdf2, + + /// __fixdfdi + Fixdfdi, + + /// __fixdfsi + Fixdfsi, + + /// __fixsfdi + Fixsfdi, + + /// __fixsfsi + Fixsfsi, + + /// __fixunsdfdi + Fixunsdfdi, + + /// __fixunsdfsi + Fixunsdfsi, + + /// __fixunssfdi + Fixunssfdi, + + /// __fixunssfsi + Fixunssfsi, + + /// __floatdidf + Floatdidf, + + /// __floatdisf + Floatdisf, + + /// __floatsidf + Floatsidf, + + /// __floatsisf + Floatsisf, + + /// __floatundidf + Floatundidf, + + /// __floatundisf + Floatundisf, + + /// __floatunsidf + Floatunsidf, + + /// __floatunsisf + Floatunsisf, + + /// __gedf2 + Gedf2, + + /// __gesf2 + Gesf2, + + /// __gtdf2 + Gtdf2, + + /// __gtsf2 + Gtsf2, + + /// __ledf2 + Ledf2, + + /// __lesf2 + Lesf2, + + /// __ltdf2 + Ltdf2, + + /// __ltsf2 + Ltsf2, + + /// __moddi3 + Moddi3, + + /// __modsi3 + Modsi3, + + /// __muldf3 + Muldf3, + + /// __muldi3 + Muldi3, + + /// __mulsf3 + Mulsf3, + + /// __mulsi3 + Mulsi3, + + /// __nedf2 + Nedf2, + + /// __negdf2 + Negdf2, + + /// __negsf2 + Negsf2, + + /// __nesf2 + Nesf2, + + /// __subdf3 + Subdf3, + + /// __subsf3 + Subsf3, + + /// __truncdfsf2 + Truncdfsf2, + + /// __udivdi3 + Udivdi3, + + /// __udivsi3 + Udivsi3, + + /// __umoddi3 + Umoddi3, + + /// __umodsi3 + Umodsi3, + + /// __unorddf2 + Unorddf2, + + /// __unordsf2 + Unordsf2, + + /// memset + Memset, + + /// sqrt + Sqrt, } impl LibCall { @@ -202,6 +364,60 @@ impl LibCall { Self::LibunwindExceptionIntoExnRef => "wasmer_vm_exception_into_exnref", Self::DebugUsize => "wasmer_vm_dbg_usize", Self::DebugStr => "wasmer_vm_dbg_str", + Self::Adddf3 => "__adddf3", + Self::Addsf3 => "__addsf3", + Self::Divdf3 => "__divdf3", + Self::Divdi3 => "__divdi3", + Self::Divsf3 => "__divsf3", + Self::Divsi3 => "__divsi3", + Self::Eqdf2 => "__eqdf2", + Self::Eqsf2 => "__eqsf2", + Self::Extendsfdf2 => "__extendsfdf2", + Self::Fixdfdi => "__fixdfdi", + Self::Fixdfsi => "__fixdfsi", + Self::Fixsfdi => "__fixsfdi", + Self::Fixsfsi => "__fixsfsi", + Self::Fixunsdfdi => "__fixunsdfdi", + Self::Fixunsdfsi => "__fixunsdfsi", + Self::Fixunssfdi => "__fixunssfdi", + Self::Fixunssfsi => "__fixunssfsi", + Self::Floatdidf => "__floatdidf", + Self::Floatdisf => "__floatdisf", + Self::Floatsidf => "__floatsidf", + Self::Floatsisf => "__floatsisf", + Self::Floatundidf => "__floatundidf", + Self::Floatundisf => "__floatundisf", + Self::Floatunsidf => "__floatunsidf", + Self::Floatunsisf => "__floatunsisf", + Self::Gedf2 => "__gedf2", + Self::Gesf2 => "__gesf2", + Self::Gtdf2 => "__gtdf2", + Self::Gtsf2 => "__gtsf2", + Self::Ledf2 => "__ledf2", + Self::Lesf2 => "__lesf2", + Self::Ltdf2 => "__ltdf2", + Self::Ltsf2 => "__ltsf2", + Self::Moddi3 => "__moddi3", + Self::Modsi3 => "__modsi3", + Self::Muldf3 => "__muldf3", + Self::Muldi3 => "__muldi3", + Self::Mulsf3 => "__mulsf3", + Self::Mulsi3 => "__mulsi3", + Self::Nedf2 => "__nedf2", + Self::Negdf2 => "__negdf2", + Self::Negsf2 => "__negsf2", + Self::Nesf2 => "__nesf2", + Self::Subdf3 => "__subdf3", + Self::Subsf3 => "__subsf3", + Self::Truncdfsf2 => "__truncdfsf2", + Self::Udivdi3 => "__udivdi3", + Self::Udivsi3 => "__udivsi3", + Self::Umoddi3 => "__umoddi3", + Self::Umodsi3 => "__umodsi3", + Self::Unorddf2 => "__unorddf2", + Self::Unordsf2 => "__unordsf2", + Self::Memset => "memset", + Self::Sqrt => "sqrt", } } } diff --git a/lib/types/src/module_hash.rs b/lib/types/src/module_hash.rs index 9025cf37f..c893115e6 100644 --- a/lib/types/src/module_hash.rs +++ b/lib/types/src/module_hash.rs @@ -51,6 +51,7 @@ impl ModuleHash { Self(hash) } + #[cfg(not(target_os = "zkvm"))] /// Generate a random [`ModuleHash`]. For when you don't care about caches. pub fn random() -> Self { let mut bytes = [0_u8; _]; diff --git a/lib/vm/Cargo.toml b/lib/vm/Cargo.toml index 0ebcadc49..fe8ab388c 100644 --- a/lib/vm/Cargo.toml +++ b/lib/vm/Cargo.toml @@ -63,6 +63,7 @@ default = [] enable-serde = ["serde", "indexmap/serde", "wasmer-types/enable-serde"] artifact-size = ["dep:loupe", "wasmer-types/artifact-size"] sanitizer = ["corosensei/sanitizer"] +force-baremetal = [] [package.metadata.docs.rs] rustc-args = ["--cfg", "docsrs"] diff --git a/lib/vm/src/libcalls.rs b/lib/vm/src/libcalls.rs index d6532e23f..d9991f208 100644 --- a/lib/vm/src/libcalls.rs +++ b/lib/vm/src/libcalls.rs @@ -560,6 +560,38 @@ pub unsafe extern "C" fn wasmer_vm_elem_drop(vmctx: *mut VMContext, elem_index: /// /// `vmctx` must be dereferenceable. #[unsafe(no_mangle)] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +pub unsafe extern "C" fn wasmer_vm_memory32_copy( + vmctx: *mut VMContext, + memory_index: u32, + dst: u64, + src: u64, + len: u64, +) { + // On RISC-V, LLVM omits `trunc` from i64 to i32, so we accept u64 and truncate ourselves. + let dst = dst as u32; + let src = src as u32; + let len = len as u32; + + unsafe { + let result = { + let memory_index = LocalMemoryIndex::from_u32(memory_index); + let instance = (*vmctx).instance(); + instance.local_memory_copy(memory_index, dst, src, len) + }; + if let Err(trap) = result { + raise_lib_trap(trap); + } + } +} + +/// Implementation of `memory.copy` for locally defined memories. +/// +/// # Safety +/// +/// `vmctx` must be dereferenceable. +#[unsafe(no_mangle)] +#[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64")))] pub unsafe extern "C" fn wasmer_vm_memory32_copy( vmctx: *mut VMContext, memory_index: u32, @@ -585,6 +617,38 @@ pub unsafe extern "C" fn wasmer_vm_memory32_copy( /// /// `vmctx` must be dereferenceable. #[unsafe(no_mangle)] +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +pub unsafe extern "C" fn wasmer_vm_imported_memory32_copy( + vmctx: *mut VMContext, + memory_index: u32, + dst: u64, + src: u64, + len: u64, +) { + // On RISC-V, LLVM omits `trunc` from i64 to i32, so we accept u64 and truncate ourselves. + let dst = dst as u32; + let src = src as u32; + let len = len as u32; + + unsafe { + let result = { + let memory_index = MemoryIndex::from_u32(memory_index); + let instance = (*vmctx).instance(); + instance.imported_memory_copy(memory_index, dst, src, len) + }; + if let Err(trap) = result { + raise_lib_trap(trap); + } + } +} + +/// Implementation of `memory.copy` for imported memories. +/// +/// # Safety +/// +/// `vmctx` must be dereferenceable. +#[unsafe(no_mangle)] +#[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64")))] pub unsafe extern "C" fn wasmer_vm_imported_memory32_copy( vmctx: *mut VMContext, memory_index: u32, @@ -955,6 +1019,76 @@ pub unsafe extern "C" fn wasmer_vm_imported_memory32_atomic_notify( } } +// A list of floating-point & 64-bit mul/div routine functions that might be +// referenced by LLVM engine. Ideally, any modern Rust / C toolchain should provide +// implementations for those functions as part of libc, so we don't need to manually +// implementation. However, it is also possible to simply implement them in Rust to +// cope with unique environments, such as __muldi3 below. +// Since we only use them by the function address value below, there is no need to +// include parameter types and return types. +unsafe extern "C" { + fn __adddf3(); + fn __addsf3(); + fn __divdf3(); + fn __divdi3(); + fn __divsf3(); + fn __divsi3(); + fn __eqdf2(); + fn __eqsf2(); + fn __extendsfdf2(); + fn __fixdfdi(); + fn __fixdfsi(); + fn __fixsfdi(); + fn __fixsfsi(); + fn __fixunsdfdi(); + fn __fixunsdfsi(); + fn __fixunssfdi(); + fn __fixunssfsi(); + fn __floatdidf(); + fn __floatdisf(); + fn __floatsidf(); + fn __floatsisf(); + fn __floatundidf(); + fn __floatundisf(); + fn __floatunsidf(); + fn __floatunsisf(); + fn __gedf2(); + fn __gesf2(); + fn __gtdf2(); + fn __gtsf2(); + fn __ledf2(); + fn __lesf2(); + fn __ltdf2(); + fn __ltsf2(); + fn __moddi3(); + fn __modsi3(); + fn __muldf3(); + fn __muldi3(); + fn __mulsf3(); + fn __nedf2(); + fn __negdf2(); + fn __negsf2(); + fn __nesf2(); + fn __subdf3(); + fn __subsf3(); + fn __truncdfsf2(); + fn __udivdi3(); + fn __udivsi3(); + fn __umoddi3(); + fn __umodsi3(); + fn __unorddf2(); + fn __unordsf2(); + fn memset(); + fn sqrt(); +} + +/// This is simply __mulsi3 but for some reasons, 64-bit environment does +/// not provide it by default. Hence we are implementing it in rust. +#[unsafe(no_mangle)] +pub extern "C" fn wasmer_vm__mulsi3(a: i32, b: i32) -> i32 { + a.wrapping_mul(b) +} + /// The function pointer to a libcall pub fn function_pointer(libcall: LibCall) -> usize { match libcall { @@ -1011,5 +1145,59 @@ pub fn function_pointer(libcall: LibCall) -> usize { } LibCall::DebugUsize => wasmer_vm_dbg_usize as *const () as usize, LibCall::DebugStr => wasmer_vm_dbg_str as *const () as usize, + LibCall::Adddf3 => __adddf3 as *const () as usize, + LibCall::Addsf3 => __addsf3 as *const () as usize, + LibCall::Divdf3 => __divdf3 as *const () as usize, + LibCall::Divdi3 => __divdi3 as *const () as usize, + LibCall::Divsf3 => __divsf3 as *const () as usize, + LibCall::Divsi3 => __divsi3 as *const () as usize, + LibCall::Eqdf2 => __eqdf2 as *const () as usize, + LibCall::Eqsf2 => __eqsf2 as *const () as usize, + LibCall::Extendsfdf2 => __extendsfdf2 as *const () as usize, + LibCall::Fixdfdi => __fixdfdi as *const () as usize, + LibCall::Fixdfsi => __fixdfsi as *const () as usize, + LibCall::Fixsfdi => __fixsfdi as *const () as usize, + LibCall::Fixsfsi => __fixsfsi as *const () as usize, + LibCall::Fixunsdfdi => __fixunsdfdi as *const () as usize, + LibCall::Fixunsdfsi => __fixunsdfsi as *const () as usize, + LibCall::Fixunssfdi => __fixunssfdi as *const () as usize, + LibCall::Fixunssfsi => __fixunssfsi as *const () as usize, + LibCall::Floatdidf => __floatdidf as *const () as usize, + LibCall::Floatdisf => __floatdisf as *const () as usize, + LibCall::Floatsidf => __floatsidf as *const () as usize, + LibCall::Floatsisf => __floatsisf as *const () as usize, + LibCall::Floatundidf => __floatundidf as *const () as usize, + LibCall::Floatundisf => __floatundisf as *const () as usize, + LibCall::Floatunsidf => __floatunsidf as *const () as usize, + LibCall::Floatunsisf => __floatunsisf as *const () as usize, + LibCall::Gedf2 => __gedf2 as *const () as usize, + LibCall::Gesf2 => __gesf2 as *const () as usize, + LibCall::Gtdf2 => __gtdf2 as *const () as usize, + LibCall::Gtsf2 => __gtsf2 as *const () as usize, + LibCall::Ledf2 => __ledf2 as *const () as usize, + LibCall::Lesf2 => __lesf2 as *const () as usize, + LibCall::Ltdf2 => __ltdf2 as *const () as usize, + LibCall::Ltsf2 => __ltsf2 as *const () as usize, + LibCall::Moddi3 => __moddi3 as *const () as usize, + LibCall::Modsi3 => __modsi3 as *const () as usize, + LibCall::Muldf3 => __muldf3 as *const () as usize, + LibCall::Muldi3 => __muldi3 as *const () as usize, + LibCall::Mulsf3 => __mulsf3 as *const () as usize, + LibCall::Mulsi3 => wasmer_vm__mulsi3 as *const () as usize, + LibCall::Nedf2 => __nedf2 as *const () as usize, + LibCall::Negdf2 => __negdf2 as *const () as usize, + LibCall::Negsf2 => __negsf2 as *const () as usize, + LibCall::Nesf2 => __nesf2 as *const () as usize, + LibCall::Subdf3 => __subdf3 as *const () as usize, + LibCall::Subsf3 => __subsf3 as *const () as usize, + LibCall::Truncdfsf2 => __truncdfsf2 as *const () as usize, + LibCall::Udivdi3 => __udivdi3 as *const () as usize, + LibCall::Udivsi3 => __udivsi3 as *const () as usize, + LibCall::Umoddi3 => __umoddi3 as *const () as usize, + LibCall::Umodsi3 => __umodsi3 as *const () as usize, + LibCall::Unorddf2 => __unorddf2 as *const () as usize, + LibCall::Unordsf2 => __unordsf2 as *const () as usize, + LibCall::Memset => memset as *const () as usize, + LibCall::Sqrt => sqrt as *const () as usize, } } diff --git a/lib/vm/src/libcalls/eh/mod.rs b/lib/vm/src/libcalls/eh/mod.rs index 9c96a599d..491023c3f 100644 --- a/lib/vm/src/libcalls/eh/mod.rs +++ b/lib/vm/src/libcalls/eh/mod.rs @@ -5,7 +5,7 @@ use crate::{InternalStoreHandle, VMContext, VMExceptionObj}; mod dwarf; cfg_if::cfg_if! { - if #[cfg(any(target_env = "msvc", target_family = "wasm"))] { + if #[cfg(any(target_env = "msvc", target_family = "wasm", target_os = "zkvm"))] { /// The implementation of Wasmer's personality function. /// /// # Safety diff --git a/lib/vm/src/mmap.rs b/lib/vm/src/mmap.rs index 75a18524f..82c172611 100644 --- a/lib/vm/src/mmap.rs +++ b/lib/vm/src/mmap.rs @@ -56,10 +56,61 @@ impl Mmap { Self::accessible_reserved(rounded_size, rounded_size, None, MmapType::Private) } + /// Allocates bytes from heap + #[cfg(target_os = "zkvm")] + pub fn accessible_reserved( + accessible_size: usize, + mapping_size: usize, + backing_file: Option, + _memory_type: MmapType, + ) -> Result { + let page_size = region::page::size(); + assert_le!(accessible_size, mapping_size); + assert_eq!(mapping_size & (page_size - 1), 0); + assert_eq!(accessible_size & (page_size - 1), 0); + + if mapping_size == 0 { + return Ok(Self::new()); + } + + assert!(backing_file.is_none()); + + use std::alloc::{Layout, alloc}; + + // mmap requires alignment to pages, we follow the same behavior + let layout = Layout::from_size_align(mapping_size, page_size).unwrap(); + let ptr = unsafe { alloc(layout) } as *const u8; + + if accessible_size < mapping_size { + unsafe { + region::protect( + ptr.add(accessible_size), + mapping_size - accessible_size, + region::Protection::NONE, + ) + } + .map_err(|e| e.to_string())?; + } + + let mut result = Self { + ptr: ptr as usize, + total_size: mapping_size, + accessible_size, + sync_on_drop: false, + }; + + if accessible_size != 0 { + // Commit the accessible size. + result.make_accessible(0, accessible_size)?; + } + + Ok(result) + } + /// Create a new `Mmap` pointing to `accessible_size` bytes of page-aligned accessible memory, /// within a reserved mapping of `mapping_size` bytes. `accessible_size` and `mapping_size` /// must be native page-size multiples. - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "zkvm"), not(target_os = "windows")))] pub fn accessible_reserved( mut accessible_size: usize, mapping_size: usize, @@ -247,10 +298,28 @@ impl Mmap { }) } + /// make_accessible for restricted environment + #[cfg(target_os = "zkvm")] + pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { + let page_size = region::page::size(); + assert_eq!(start & (page_size - 1), 0); + assert_eq!(len & (page_size - 1), 0); + assert_le!(len, self.total_size); + assert_le!(start, self.total_size - len); + + unsafe { + let ptr_start = (self.ptr as *mut u8).add(start); + region::protect(ptr_start, len, region::Protection::READ_WRITE) + .map_err(|e| e.to_string())?; + ptr::write_bytes(ptr_start, 0, len); + } + Ok(()) + } + /// Make the memory starting at `start` and extending for `len` bytes accessible. /// `start` and `len` must be native page-size multiples and describe a range within /// `self`'s reserved memory. - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "zkvm"), not(target_os = "windows")))] pub fn make_accessible(&mut self, start: usize, len: usize) -> Result<(), String> { let page_size = region::page::size(); assert_eq!(start & (page_size - 1), 0); @@ -371,7 +440,26 @@ impl Mmap { } impl Drop for Mmap { - #[cfg(not(target_os = "windows"))] + #[cfg(target_os = "zkvm")] + fn drop(&mut self) { + if self.total_size != 0 { + unsafe { + region::protect( + self.ptr as *const u8, + self.total_size, + region::Protection::READ_WRITE, + ) + } + .expect("restore memory as accessible again"); + + use std::alloc::{Layout, dealloc}; + + let layout = Layout::from_size_align(self.total_size, region::page::size()).unwrap(); + unsafe { dealloc(self.ptr as *mut u8, layout) } + } + } + + #[cfg(all(not(target_os = "zkvm"), not(target_os = "windows")))] fn drop(&mut self) { if self.total_size != 0 { if self.sync_on_drop { diff --git a/lib/vm/src/trap/mod.rs b/lib/vm/src/trap/mod.rs index 5cee47b81..992340723 100644 --- a/lib/vm/src/trap/mod.rs +++ b/lib/vm/src/trap/mod.rs @@ -6,6 +6,10 @@ #[allow(clippy::module_inception)] mod trap; +#[cfg(not(any(target_os = "zkvm", feature = "force-baremetal")))] +mod traphandlers; +#[cfg(any(target_os = "zkvm", feature = "force-baremetal"))] +#[path = "traphandlers_baremetal.rs"] mod traphandlers; pub use trap::Trap; @@ -13,5 +17,7 @@ pub use traphandlers::{ TrapHandlerFn, VMConfig, catch_traps, on_host_stack, raise_lib_trap, raise_user_trap, set_stack_size, wasmer_call_trampoline, }; +#[cfg(any(target_os = "zkvm", feature = "force-baremetal"))] +pub use traphandlers::{UnwindReason, install_unwinder}; pub use traphandlers::{init_traps, resume_panic}; pub use wasmer_types::TrapCode; diff --git a/lib/vm/src/trap/traphandlers_baremetal.rs b/lib/vm/src/trap/traphandlers_baremetal.rs new file mode 100644 index 000000000..abad6d2a6 --- /dev/null +++ b/lib/vm/src/trap/traphandlers_baremetal.rs @@ -0,0 +1,118 @@ +use crate::vmcontext::{VMFunctionContext, VMTrampoline}; +use crate::{Trap, VMContext, VMFunctionBody}; +use std::any::Any; +use std::error::Error; +use std::mem; + +/// Dummy trap handler type for baremetal mode +pub type TrapHandlerFn<'a> = (); +/// Dummy config type for baremetal mode +pub struct VMConfig { + /// This is put here to make compile happy, we don't really use this + /// value. + pub wasm_stack_size: Option, +} + +/// Baremetal does not support setting stack size, the function is kept +/// here to preserve APIs +pub fn set_stack_size(_size: usize) { + panic!("Setting stack size is not supported in baremetal feature!"); +} + +/// In baremetal mode, init_traps does nothing +pub fn init_traps() {} + +/// In baremetal mode, on_host_stack does not addition action +pub fn on_host_stack T, T>(f: F) -> T { + f() +} + +/// In baremetal mode, catch_traps simply ignores trap handler and config +pub unsafe fn catch_traps( + _trap_handler: Option<*const TrapHandlerFn<'static>>, + _config: &VMConfig, + closure: F, +) -> Result +where + F: FnOnce() -> R + 'static, +{ + Ok(closure()) +} + +/// Unwinding reason +#[derive(Debug)] +pub enum UnwindReason { + /// A panic caused by the host + Panic(Box), + /// A custom error triggered by the user + UserTrap(Box), + /// A Trap triggered by a wasm libcall + LibTrap(Trap), +} + +impl UnwindReason { + /// Build a Trap from UnwindReason + pub fn into_trap(self) -> Trap { + match self { + Self::UserTrap(data) => Trap::User(data), + Self::LibTrap(trap) => trap, + Self::Panic(panic) => std::panic::resume_unwind(panic), + } + } +} + +static mut UNWINDER: Option> = None; + +/// Install a custom unwinder to use +pub fn install_unwinder(unwinder: Option>) { + unsafe { UNWINDER = unwinder }; +} + +unsafe fn unwind_with(reason: UnwindReason) -> ! { + println!("Unwinding: {reason:?}"); + if let Some(unwinder) = unsafe { &*&raw const UNWINDER } { + unwinder(reason); + unreachable!() + } else { + panic!("Unwinding: {reason:?}"); + } +} + +/// Unwind with Rust panic +pub unsafe fn resume_panic(payload: Box) -> ! { + unsafe { unwind_with(UnwindReason::Panic(payload)) } +} + +/// Raise user trap +pub unsafe fn raise_user_trap(data: Box) -> ! { + unsafe { unwind_with(UnwindReason::UserTrap(data)) } +} + +/// Raise library trap +pub unsafe fn raise_lib_trap(trap: Trap) -> ! { + unsafe { unwind_with(UnwindReason::LibTrap(trap)) } +} + +/// When the inner functions have been mocked, wasmer_call_trampoline +/// in baremetal mode can be implemented exactly as it is in os mode. +pub unsafe fn wasmer_call_trampoline( + trap_handler: Option<*const TrapHandlerFn<'static>>, + config: &VMConfig, + vmctx: VMFunctionContext, + trampoline: VMTrampoline, + callee: *const VMFunctionBody, + values_vec: *mut u8, +) -> Result<(), Trap> { + unsafe { + catch_traps(trap_handler, config, move || { + mem::transmute::< + unsafe extern "C" fn( + *mut VMContext, + *const VMFunctionBody, + *mut wasmer_types::RawValue, + ), + extern "C" fn(VMFunctionContext, *const VMFunctionBody, *mut u8), + >(trampoline)(vmctx, callee, values_vec); + }) + } +}