diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39608220c..8cfc76e63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,6 +320,7 @@ jobs: wasm_gc_effect_arg_overflow_regression \ wasm_gc_games_differential_mir \ wasm_gc_handler_carrier \ + wasm_gc_map_versions_spec \ wasm_gc_optimize_trunc_sat \ wasm_gc_packed_sequence \ wasm_gc_perslot_int_unboxing_differential \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 3290c92c3..07568de6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ The generated loop is now written from the program's source alone, and the manif ### Fixed +- **wasm-gc: `Map.set` no longer copies the map, and `Map.remove` no longer changes the map it was given.** `set` copied every bucket unless the compiler could prove the map had no other holder, which it cannot for a map held in a record field such as an answer module's state. A 100 000-entry state map served 2000 `Map.set` requests in 11 s under `aver run --wasm-gc` and 1.9 s on Node 26. `remove` wrote into the map it was given, so a caller that still held that map saw the key gone. Both now write into the map's arrays in place and return a new version. The version they were given stays valid, because a record of what the write replaced is kept with it. The same run now takes 0.4 s under `aver run --wasm-gc` and 0.3 s on Node. Reading an older version again costs one step for each write made since. - **A program with the generated loop can key its own waits by a type of its own.** The loop keys its wait by `Int`, and a hand-written `Wait.poll` keyed by a sum beside it used to be refused by the Rust door ("this program keys one wait set by 'Int' and another by 'Watch'") and to fail wasm-gc validation. In a program that answers a capability of its own, each such wait, in the entry or in a dependency, now goes through helpers generated for its key type that carry it through an `Int`-keyed wait. It answers the same keys in the same order. Its recording holds the `Int`-keyed wait. The wasm-gc wait ABI is unchanged, and `--target wasip2` runs such waits too. A program that answers no capability still keys all its waits one way. - **`check` no longer asks a verify block of a function no verify case can call.** A parameter of a capability resource type (`Tcp.Connection`, `Work.Job`, a job kind's handle), or of a tuple, record or sum of the module that always carries one, has no value a case can write, so such a pure branching helper failed `error[missing-verify]` with no way to satisfy it. It is now exempt, the way effectful functions are. A parameter with an empty value (`List`, `Option`, a sum with a resource-free variant) still needs its verify block. - **A `main` that answers `Err` exits non-zero on wasm-gc and wasip2**, with the error on stderr on wasm-gc, as it already did on the VM and in generated Rust. Both wasm targets used to exit zero. diff --git a/src/codegen/wasm_gc/body/builtins_wasip2.rs b/src/codegen/wasm_gc/body/builtins_wasip2.rs index 267dfa0d8..67e9748a4 100644 --- a/src/codegen/wasm_gc/body/builtins_wasip2.rs +++ b/src/codegen/wasm_gc/body/builtins_wasip2.rs @@ -1004,6 +1004,9 @@ fn emit_poll_wasip2( WasmGcError::Validation(format!("{operation} on wasip2: helper fn idx missing")) })?; emit_mir_expr(func, &args[0], slots, ctx)?; + // The poll helper walks the map's buckets directly: hand it the + // current version. + super::from_mir::emit_map_arg_current(func, &args[0], ctx)?; emit_mir_expr(func, &args[1], slots, ctx)?; func.instruction(&Instruction::Call(helper)); Ok(()) @@ -1750,6 +1753,7 @@ fn emit_http_simple_method_wasip2( func.instruction(&Instruction::ArrayNewDefault(map_slots.values_array)); func.instruction(&Instruction::I32Const(INITIAL_CAP)); func.instruction(&Instruction::ArrayNewDefault(map_slots.hashes_array)); + crate::codegen::wasm_gc::maps::emit_no_diff(func, map_slots); func.instruction(&Instruction::StructNew(map_slots.map)); func.instruction(&Instruction::Call(fn_idx)); Ok(()) diff --git a/src/codegen/wasm_gc/body/from_mir/builtins.rs b/src/codegen/wasm_gc/body/from_mir/builtins.rs index f597acb0e..98bc9402c 100644 --- a/src/codegen/wasm_gc/body/from_mir/builtins.rs +++ b/src/codegen/wasm_gc/body/from_mir/builtins.rs @@ -1966,11 +1966,34 @@ pub(crate) fn emit_mir_result_from_option( Ok(MirBuiltinEmit::Produced(true)) } +/// With a `Map` argument of `arg`'s type on the stack, make it the current +/// version of its map (`maps/versions.rs`). A host import reads the map's +/// buckets directly and cannot do it itself. Any other type is left alone. +pub(crate) fn emit_map_arg_current( + func: &mut Function, + arg: &Spanned, + ctx: &EmitCtx<'_>, +) -> Result<(), WasmGcError> { + let aver = aver_type_str_of(arg); + let canonical: String = aver.chars().filter(|c| !c.is_whitespace()).collect(); + if !canonical.starts_with("Map<") { + return Ok(()); + } + let helpers = ctx + .fn_map + .map_helpers_lookup(&canonical) + .ok_or(WasmGcError::Validation(format!( + "a `{aver}` argument crosses to the host but no Map helpers are registered for it" + )))?; + func.instruction(&Instruction::Call(helpers.reroot)); + Ok(()) +} + /// Mirror of `emit_map_kv_call`: the `Map.*` methods dispatch to the /// per-`Map` helpers (`fn_map.map_helpers_lookup`). `has` reuses -/// the `get_pair` helper and drops the value; `set` picks `set_in_place` -/// vs the clone-on-write `set` by `mir_arg_uniquely_owned` (the MIR -/// analogue of the oracle's `arg_uniquely_owned`). The canonical comes +/// the `get_pair` helper and drops the value. `set` and `remove` need no +/// ownership fact: they write in place and leave the map they were given +/// valid as an older version (`maps/versions.rs`). The canonical comes /// from the map arg's stamped type; every arg recurses `emit_mir_expr`. pub(crate) fn emit_mir_map_builtin( func: &mut Function, @@ -2062,13 +2085,7 @@ pub(crate) fn emit_mir_map_builtin( "Map.{method}: map argument has type `{map_aver}` but no helpers are registered" )))?; match method { - "set" => { - if mir_arg_uniquely_owned(&args[0], ctx) { - helpers.set_in_place - } else { - helpers.set - } - } + "set" => helpers.set, "get" => helpers.get, "len" => helpers.len, "keys" => helpers.keys, diff --git a/src/codegen/wasm_gc/body/from_mir/mod.rs b/src/codegen/wasm_gc/body/from_mir/mod.rs index 7df5a2674..81f8385af 100644 --- a/src/codegen/wasm_gc/body/from_mir/mod.rs +++ b/src/codegen/wasm_gc/body/from_mir/mod.rs @@ -84,6 +84,7 @@ pub(super) use super::slots::count_value_params; pub(super) use super::{CallerFnCollector, EmitCtx, FnMap, SlotTable, Wasip2Lowering}; mod builtins; +pub(super) use builtins::emit_map_arg_current; mod collections; mod constructors; mod control; @@ -922,6 +923,10 @@ pub(crate) fn emit_mir_expr( if emit_mir_expr(func, arg, slots, ctx)?.is_none() { return Ok(None); } + // The host reads a map's buckets directly + // (`Tcp.poll`, `Wait.poll`), so hand it the + // current version. + builtins::emit_map_arg_current(func, arg, ctx)?; if ctx.registry.bignum && int_args.contains(&i) { // CHECKED (not saturating): an out-of-i64 Int // crossing the host effect boundary must diff --git a/src/codegen/wasm_gc/maps.rs b/src/codegen/wasm_gc/maps.rs index 28797a66e..f745921b6 100644 --- a/src/codegen/wasm_gc/maps.rs +++ b/src/codegen/wasm_gc/maps.rs @@ -16,18 +16,25 @@ //! mut (ref null $keys_array) keys; //! mut (ref null $values_array) values; //! mut (ref null $hashes_array) hashes; +//! mut (ref null $diff_KV) diff; //! } //! ``` //! +//! `set` and `remove` write into the arrays in place and return a new +//! struct over them; the map they were given stays a valid value through +//! the `diff` the write leaves on it. Several versions of one map share +//! its arrays, and a helper makes the version it was handed current +//! (`reroot`) before it reads them. See `maps/versions.rs`. +//! //! Empty slot marker = `keys[i] == null` (only valid for `K` that //! cannot legitimately be null, which Aver guarantees for ref types //! since the type system rejects null at the source level). //! -//! The table grows. Both insert helpers check, before they probe, -//! whether the entry they are about to add would push occupancy past +//! The table grows. The insert helper checks, before it probes, +//! whether the entry it is about to add would push occupancy past //! three quarters of capacity (see [`LOAD_SHIFT`]); when it would, -//! they allocate keys/values/hashes arrays at twice the capacity, rehash every -//! live entry into the wider mask using its cached hash, and probe there +//! it allocates keys/values/hashes arrays at twice the capacity, rehashes every +//! live entry into the wider mask using its cached hash, and probes there //! instead. `Map.set` //! is therefore total up to memory exhaustion, the same promise the //! VM and the Rust backend make. Capacity is per-map state in the @@ -37,7 +44,7 @@ //! What the growth buys is an invariant: **a map never fills**. After //! any insert, occupancy is at most three quarters of capacity, so //! some slot is always free and every probe loop terminates on it. -//! The two insert helpers still carry their wrap guard, +//! The insert helper still carries its wrap guard, //! and it is now a backstop rather than a limit — reaching it means //! the growth above it did not happen, which is a compiler bug, and //! trapping is how that bug surfaces instead of hanging. The rehash @@ -69,6 +76,10 @@ use super::types::{MapSlots, TypeRegistry}; use super::wat_helper; mod order; +mod versions; + +use versions::{VersionedWrite, emit_versioned_write}; +pub(super) use versions::{emit_no_diff, emit_reroot_local}; /// Initial bucket count — power of two so masking with `cap-1` /// instead of `i32.rem_u` works, and every doubling keeps it one. @@ -78,9 +89,7 @@ mod order; /// fresh header map per `Http.get` call and per inbound request. /// Sixteen buckets cost two 16-element arrays; the old fixed 16384 /// cost two 16384-element ones, roughly 128 KB of zeroes per empty -/// map. It also bounds the clone-on-write `set`, which copies `cap` -/// slots on every call — a three-key map used to pay 16384 element -/// copies per insert. Doubling makes the total cost of reaching `n` +/// map. Doubling makes the total cost of reaching `n` /// entries linear in `n`, so starting small costs a few extra /// rehashes early and nothing after that. /// @@ -124,15 +133,15 @@ pub(super) struct KeyHelpers { #[derive(Debug, Clone, Copy)] pub(super) struct MapKVHelpers { pub(super) empty: u32, - /// Clone-on-write `set` — allocates fresh keys/values/hashes arrays, - /// `array.copy`s them, mutates the copies. Used when `ir::alias` - /// flags the call site's map slot as alias-prone. + /// `set(m, k, v) -> m'`. Writes the bucket in place and returns a new + /// version over the same arrays; `m` stays valid through the diff the + /// write leaves on it (see `maps/versions.rs`). A grow writes into new + /// arrays instead and leaves `m` untouched. pub(super) set: u32, - /// In-place `set` — probes the source map's keys/values/hashes arrays - /// directly, `array.set`s into them, returns a struct.new wrapping - /// the same arrays with the updated size. Sound only when the IR - /// alias pass + last-use proves the map slot is uniquely owned. - pub(super) set_in_place: u32, + /// `reroot(m) -> m`. Makes `m` the version that owns its arrays' + /// contents. Every helper that reads a map's arrays calls it first, and + /// so must any code outside this module that reads them. + pub(super) reroot: u32, pub(super) get: u32, pub(super) len: u32, /// `get_or_default(m, k, default) -> V`. Fused shape that backs @@ -157,12 +166,11 @@ pub(super) struct MapKVHelpers { pub(super) keys: u32, /// `values(m) -> List` in the same key-derived order as `keys`. pub(super) values: u32, - /// `remove(m, k) -> m`. Linear-probe locate of `k`, then a + /// `remove(m, k) -> m'`. Linear-probe locate of `k`, then a /// backwards-shift scan over the rest of the probe run so every /// entry that probed past the emptied slot is still found by a - /// later `get` — see `emit_map_remove`. Mutates `m` in place - /// and returns the same handle (Aver semantics: same shape as - /// `set`, the returned ref is structurally equal). + /// later `get` — see `emit_map_remove`. Each bucket it writes is a + /// versioned write, so `m` keeps its entries, as it does for `set`. pub(super) remove: u32, /// `entries(m) -> List>` in canonical key order. pub(super) entries: u32, @@ -206,12 +214,8 @@ pub(super) struct MapHelperRegistry { #[derive(Debug, Clone, Copy)] struct MapKVTypeIdx { empty: u32, - /// Same wasm-fn type as `set_in_place` — `(map, k, v) -> map`. - /// Two distinct type entries because the fn-type table is - /// indexed by type-idx, not shape, and `assign_slots` writes - /// each helper to its own slot. set: u32, - set_in_place: u32, + reroot: u32, get: u32, len: u32, get_or_default: u32, @@ -537,7 +541,7 @@ impl MapHelperRegistry { *next_type_idx += 1; let set_type_idx = *next_type_idx; *next_type_idx += 1; - let set_in_place_type_idx = *next_type_idx; + let reroot_type_idx = *next_type_idx; *next_type_idx += 1; let get_type_idx = *next_type_idx; *next_type_idx += 1; @@ -569,7 +573,7 @@ impl MapHelperRegistry { *next_wasm_fn_idx += 1; let set_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; - let set_in_place_fn = *next_wasm_fn_idx; + let reroot_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; let get_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; @@ -632,7 +636,7 @@ impl MapHelperRegistry { MapKVHelpers { empty: empty_fn, set: set_fn, - set_in_place: set_in_place_fn, + reroot: reroot_fn, get: get_fn, len: len_fn, get_or_default: god_fn, @@ -653,7 +657,7 @@ impl MapHelperRegistry { MapKVTypeIdx { empty: empty_type_idx, set: set_type_idx, - set_in_place: set_in_place_type_idx, + reroot: reroot_type_idx, get: get_type_idx, len: len_type_idx, get_or_default: god_type_idx, @@ -682,9 +686,9 @@ impl MapHelperRegistry { self.kv.get(canonical).copied() } - /// `(wasm fn idx, name)` for the two insert helpers of every - /// registered `Map`, ascending by index — the shape the - /// `name` section's function subsection wants. + /// `(wasm fn idx, name)` for the insert helper of every registered + /// `Map`, ascending by index — the shape the `name` section's + /// function subsection wants. /// /// These are the only helpers that can trap, and they can only do /// it on a broken invariant: the table grows before it fills, so a @@ -693,7 +697,7 @@ impl MapHelperRegistry { /// which map's insert stopped and that a stop there is a bug, and /// the engine's backtrace reads it back. pub(super) fn capacity_helper_names(&self) -> Vec<(u32, String)> { - let mut named = Vec::with_capacity(self.kv_order.len() * 2); + let mut named = Vec::with_capacity(self.kv_order.len()); for canonical in &self.kv_order { let Some(h) = self.kv.get(canonical) else { continue; @@ -702,12 +706,6 @@ impl MapHelperRegistry { h.set, format!("{CAPACITY_HELPER_NAME_PREFIX}{canonical} (table grows; a stop here is a resize bug)"), )); - named.push(( - h.set_in_place, - format!( - "{CAPACITY_HELPER_NAME_PREFIX}{canonical} in place (table grows; a stop here is a resize bug)" - ), - )); } named.sort_by_key(|(idx, _)| *idx); named @@ -777,10 +775,10 @@ impl MapHelperRegistry { // empty : () -> Map types.ty().function([], [map_ref]); - // set : (Map, K, V) -> Map (clone-on-write) - types.ty().function([map_ref, k_val, v_val], [map_ref]); - // set_in_place : (Map, K, V) -> Map (alias-free fast path) + // set : (Map, K, V) -> Map types.ty().function([map_ref, k_val, v_val], [map_ref]); + // reroot : (Map) -> Map + types.ty().function([map_ref], [map_ref]); // get : (Map, K) -> Option types.ty().function([map_ref, k_val], [opt_ref]); // len : (Map) -> i64 @@ -874,7 +872,7 @@ impl MapHelperRegistry { let t = self.kv_type_indices[canonical]; funcs.function(t.empty); funcs.function(t.set); - funcs.function(t.set_in_place); + funcs.function(t.reroot); funcs.function(t.get); funcs.function(t.len); funcs.function(t.get_or_default); @@ -977,28 +975,47 @@ impl MapHelperRegistry { )) })?; let helpers = self.kv[canonical]; + let reroot = helpers.reroot; codes.function(&emit_map_empty(canonical, registry)?); - codes.function(&emit_map_set(canonical, registry, key_h)?); - codes.function(&emit_map_set_in_place(canonical, registry, key_h)?); - codes.function(&emit_map_get(canonical, registry, key_h)?); + codes.function(&emit_map_set(canonical, registry, key_h, reroot)?); + codes.function(&versions::emit_map_reroot( + canonical, + registry, + slots_for(canonical, registry)?, + )?); + codes.function(&emit_map_get(canonical, registry, key_h, reroot)?); codes.function(&emit_map_len(canonical, registry)?); - codes.function(&emit_map_get_or_default(canonical, registry, key_h)?); - codes.function(&emit_map_get_pair(canonical, registry, key_h)?); + codes.function(&emit_map_get_or_default( + canonical, registry, key_h, reroot, + )?); + codes.function(&emit_map_get_pair(canonical, registry, key_h, reroot)?); codes.function(&emit_map_order_sift(canonical, registry, cmp_fn)?); codes.function(&emit_map_order_slots( canonical, registry, helpers.order_sift, + reroot, + )?); + codes.function(&emit_map_keys( + canonical, + registry, + helpers.order_slots, + reroot, )?); - codes.function(&emit_map_keys(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_values(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_remove(canonical, registry, key_h)?); - codes.function(&emit_map_entries(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_from_list( + codes.function(&emit_map_values( canonical, registry, - helpers.set_in_place, + helpers.order_slots, + reroot, )?); + codes.function(&emit_map_remove(canonical, registry, key_h, reroot)?); + codes.function(&emit_map_entries( + canonical, + registry, + helpers.order_slots, + reroot, + )?); + codes.function(&emit_map_from_list(canonical, registry, helpers.set)?); // Structural eq + commutative hash for `Map`. V's // hash + eq fn idxs come from `all_key_helpers` (same // table that drives K dispatch — V is just another @@ -1011,8 +1028,11 @@ impl MapHelperRegistry { key_h, v_helpers, helpers.get, + reroot, + )?); + codes.function(&emit_map_hash( + canonical, registry, key_h, v_helpers, reroot, )?); - codes.function(&emit_map_hash(canonical, registry, key_h, v_helpers)?); } Ok(()) } @@ -1499,65 +1519,27 @@ fn emit_map_empty(canonical: &str, registry: &TypeRegistry) -> Result map`. Linear-probing open-addressing insert over -/// a clone of the map's arrays. Returns a fresh map struct wrapping -/// them; the input map is never observed mutated. +/// `set(map, k, v) -> map'`. Linear-probing open-addressing insert. +/// +/// Settles which arrays to probe (growing first if this entry would +/// overfill the table), then linear-probes from the key's home bucket — +/// an empty slot inserts, a matching key updates. Into the map's own +/// arrays the write is a versioned one (`maps/versions.rs`): the bucket's +/// old contents go on `map`'s diff and the result is a new version over the +/// same arrays, so `map` is unchanged as a value and nothing is copied. A +/// grow has already moved every entry into new arrays, which only the +/// result sees, so it writes into them directly. fn emit_map_set( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, -) -> Result { - emit_map_insert(canonical, registry, keyh, TableSource::Clone) -} - -/// `set_in_place(map, k, v) -> map`. Same insert as [`emit_map_set`] -/// without the entry-time copy of `keys` / `values` / `hashes` — the caller has -/// proven the map's arrays are uniquely owned. The returned struct -/// still re-wraps the arrays with the updated size; callers expect a -/// fresh map handle either way, which is also what lets a grow swap -/// the arrays out from under the old one. -fn emit_map_set_in_place( - canonical: &str, - registry: &TypeRegistry, - keyh: KeyHelpers, -) -> Result { - emit_map_insert(canonical, registry, keyh, TableSource::Owned) -} - -/// The body both insert helpers share: settle which arrays to probe -/// (growing first if this entry would overfill the table), then -/// linear-probe from the key's home bucket — empty slot inserts, -/// matching key updates. The two differ only in [`TableSource`], which -/// the prologue reads; every instruction after it is the same, so it -/// is written once. -fn emit_map_insert( - canonical: &str, - registry: &TypeRegistry, - keyh: KeyHelpers, - source: TableSource, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -1577,17 +1559,25 @@ fn emit_map_insert( nullable: true, heap_type: HeapType::Concrete(slots.hashes_array), }); - // params: 0=map, 1=k, 2=v + let map_ref = ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }); + let diff_ref = ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.diff), + }); + // params: 0=map (then the new version, after a versioned write), + // 1=k, 2=v // locals: 3=cap, 4=mask, 5=idx, 6=keys, 7=values, 8=cur_key, // 9=home, 10=src_keys, 11=src_values, 12=i, - // 13=hashes, 14=src_hashes, 15=query_hash + // 13=hashes, 14=src_hashes, 15=query_hash, 16=diff, 17=next // - // 6/7 are the arrays the probe writes to — a clone, the map's own, - // or a wider set a grow just filled. 10/11 are the map's arrays as - // handed in, which the grow rehashes out of and the clone copies - // from. 13/14 are their parallel cached-hash arrays. 12 walks them; - // 15 carries a stored hash during growth, then the query hash. - // 5/8/9 are reused by both probe loops. + // 6/7/13 are the arrays the probe writes to — the map's own, or a + // wider set a grow just filled. 10/11/14 are the map's arrays as + // handed in, which the grow rehashes out of. 12 walks them; 15 + // carries a stored hash during growth, then the query hash. 5/8/9 are + // reused by both probe loops. let mut f = Function::new([ (1, ValType::I32), // 3: cap (1, ValType::I32), // 4: mask @@ -1598,13 +1588,16 @@ fn emit_map_insert( (1, ValType::I32), // 9: home (probe start bucket) (1, keys_ref), // 10: src_keys (the map's own) (1, values_ref), // 11: src_values (the map's own) - (1, ValType::I32), // 12: i (rehash / copy cursor) + (1, ValType::I32), // 12: i (rehash cursor) (1, hashes_ref), // 13: hashes (probe target) (1, hashes_ref), // 14: src_hashes (the map's own) (1, ValType::I32), // 15: stored/query hash + (1, diff_ref), // 16: diff + (1, map_ref), // 17: next version ]); - emit_insert_prologue(&mut f, slots, source); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); + emit_insert_prologue(&mut f, slots); // query_hash = hash(k); idx = query_hash & mask; home = idx f.instruction(&Instruction::LocalGet(1)); @@ -1632,36 +1625,25 @@ fn emit_map_insert( f.instruction(&Instruction::LocalGet(8)); f.instruction(&Instruction::RefIsNull); f.instruction(&Instruction::If(BlockType::Empty)); - // keys[idx] = box(k) (primitive K) or k (ref K) - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(1)); - emit_box_key(&mut f, k_aver, registry); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - // values[idx] = v - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(2)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - // hashes[idx] = query_hash - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(15)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); - // return struct.new $map (map.size + 1, cap, keys, values, hashes) - f.instruction(&Instruction::LocalGet(0)); - f.instruction(&Instruction::StructGet { - struct_type_index: slots.map, - field_index: 0, - }); - f.instruction(&Instruction::I32Const(1)); - f.instruction(&Instruction::I32Add); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::StructNew(slots.map)); - f.instruction(&Instruction::Return); + let insert = |f: &mut Function| { + // keys[idx] = box(k) (primitive K) or k (ref K) + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(1)); + emit_box_key(f, k_aver, registry); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + // values[idx] = v + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + // hashes[idx] = query_hash + f.instruction(&Instruction::LocalGet(13)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(15)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }; + emit_insert_write(&mut f, slots, 1, insert); f.instruction(&Instruction::End); // else if cached_hash == query_hash && eq(unbox(cur_key), k): update @@ -1678,22 +1660,13 @@ fn emit_map_insert( f.instruction(&Instruction::LocalGet(1)); f.instruction(&Instruction::Call(keyh.eq)); f.instruction(&Instruction::If(BlockType::Empty)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(2)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - // return struct.new $map (map.size, cap, keys, values, hashes) - f.instruction(&Instruction::LocalGet(0)); - f.instruction(&Instruction::StructGet { - struct_type_index: slots.map, - field_index: 0, - }); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::StructNew(slots.map)); - f.instruction(&Instruction::Return); + let update = |f: &mut Function| { + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + }; + emit_insert_write(&mut f, slots, 0, update); f.instruction(&Instruction::End); f.instruction(&Instruction::End); @@ -1713,15 +1686,81 @@ fn emit_map_insert( Ok(f) } +/// Write bucket `idx` (local 5) of an insert with `write`, then return the +/// map it makes, `size_delta` entries larger than `map`. +/// +/// When the probe wrote into `map`'s own arrays (6 is 10), the write is a +/// versioned one and the result is the new version. After a grow the +/// arrays are new and only the result holds them, so it writes directly. +fn emit_insert_write( + f: &mut Function, + slots: MapSlots, + size_delta: i32, + write: impl Fn(&mut Function), +) { + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(10)); + f.instruction(&Instruction::RefEq); + f.instruction(&Instruction::If(BlockType::Empty)); + emit_versioned_write( + f, + slots, + &VersionedWrite { + version: 0, + diff: 16, + next: 17, + idx: 5, + cap: 3, + keys: 6, + values: 7, + hashes: 13, + }, + &write, + ); + if size_delta != 0 { + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + f.instruction(&Instruction::I32Const(size_delta)); + f.instruction(&Instruction::I32Add); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: 0, + }); + } + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::Return); + f.instruction(&Instruction::End); + write(f); + // return struct.new $map (map.size + delta, cap, keys, values, hashes) + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + if size_delta != 0 { + f.instruction(&Instruction::I32Const(size_delta)); + f.instruction(&Instruction::I32Add); + } + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(13)); + emit_no_diff(f, slots); + f.instruction(&Instruction::StructNew(slots.map)); + f.instruction(&Instruction::Return); +} + /// Settle `cap` (3), `mask` (4) and the arrays the probe will write to /// (6, 7, 13) for one insert, growing the table first when the entry about /// to be added would push occupancy past three quarters of capacity. /// -/// Growing and copying are the same act, so they are one branch each -/// and never both: a clone-on-write insert that grows allocates the -/// wider arrays and rehashes straight into them, which copies every -/// live entry exactly once. The `array.copy` in the other branch is -/// the cheaper move for the far more common insert that does not grow. +/// A grow allocates the wider arrays and rehashes straight into them, +/// which copies every live entry exactly once. Without a grow the probe +/// writes into the map's own arrays (see [`emit_insert_write`]). /// /// Rehashing rather than copying is what the wider mask requires: a /// key's bucket is `hash & (cap - 1)`, so doubling `cap` exposes one @@ -1732,7 +1771,7 @@ fn emit_map_insert( /// The map's own arrays are read out first (10, 11, 14) because both /// branches need them, and because reading them before the branch /// keeps the grow path from re-reading struct fields per entry. -fn emit_insert_prologue(f: &mut Function, slots: MapSlots, source: TableSource) { +fn emit_insert_prologue(f: &mut Function, slots: MapSlots) { // cap = map.cap; src_keys/src_values/src_hashes = map arrays f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { @@ -1891,57 +1930,12 @@ fn emit_insert_prologue(f: &mut Function, slots: MapSlots, source: TableSource) f.instruction(&Instruction::I32Const(1)); f.instruction(&Instruction::I32Sub); f.instruction(&Instruction::LocalSet(4)); - match source { - TableSource::Clone => { - // keys = array.new_default cap; array.copy keys 0 src_keys 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.keys_array)); - f.instruction(&Instruction::LocalSet(6)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(10)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.keys_array, - array_type_index_src: slots.keys_array, - }); - // values = array.new_default cap; array.copy values 0 src_values 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.values_array)); - f.instruction(&Instruction::LocalSet(7)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(11)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.values_array, - array_type_index_src: slots.values_array, - }); - // hashes = array.new_default cap; array.copy hashes 0 src_hashes 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.hashes_array)); - f.instruction(&Instruction::LocalSet(13)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.hashes_array, - array_type_index_src: slots.hashes_array, - }); - } - TableSource::Owned => { - f.instruction(&Instruction::LocalGet(10)); - f.instruction(&Instruction::LocalSet(6)); - f.instruction(&Instruction::LocalGet(11)); - f.instruction(&Instruction::LocalSet(7)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalSet(13)); - } - } + f.instruction(&Instruction::LocalGet(10)); + f.instruction(&Instruction::LocalSet(6)); + f.instruction(&Instruction::LocalGet(11)); + f.instruction(&Instruction::LocalSet(7)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalSet(13)); f.instruction(&Instruction::End); // grow if/else } @@ -2009,6 +2003,7 @@ fn emit_map_get( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2052,6 +2047,7 @@ fn emit_map_get( ]); let _ = k_val; // cap, mask, keys, values + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2156,6 +2152,7 @@ fn emit_map_get_or_default( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2196,6 +2193,7 @@ fn emit_map_get_or_default( ]); // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2298,6 +2296,7 @@ fn emit_map_get_pair( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2337,6 +2336,7 @@ fn emit_map_get_pair( (1, ValType::I32), // 10: query_hash ]); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2819,6 +2819,7 @@ fn emit_map_order_slots( canonical: &str, registry: &TypeRegistry, sift_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let order_idx = registry @@ -2837,6 +2838,7 @@ fn emit_map_order_slots( // params: 0=map. locals: 1=keys, 2=indices, 3=count, 4=cap, // 5=slot, 6=used, 7=start, 8=end, 9=tmp. let mut f = Function::new([(1, keys_ref), (1, order_ref), (7, ValType::I32)]); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2966,6 +2968,7 @@ fn emit_map_keys( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, _) = super::types::parse_map_kv(canonical).unwrap(); @@ -2975,7 +2978,7 @@ fn emit_map_keys( .ok_or(WasmGcError::Validation(format!( "Map.keys: `{list_canonical}` not registered" )))?; - emit_map_walk_keys_to_list(slots, list_idx, k_aver, registry, order_slots_fn) + emit_map_walk_keys_to_list(slots, list_idx, k_aver, registry, order_slots_fn, reroot_fn) } /// `values(m) -> List`. Same shape as `keys` but pulls from @@ -2984,6 +2987,7 @@ fn emit_map_values( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (_, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2993,7 +2997,7 @@ fn emit_map_values( .ok_or(WasmGcError::Validation(format!( "Map.values: `{list_canonical}` not registered" )))?; - emit_map_walk_values_to_list(slots, registry, list_idx, order_slots_fn) + emit_map_walk_values_to_list(slots, registry, list_idx, order_slots_fn, reroot_fn) } /// Real impl for `Map.keys` walking the keys array. Per primitive @@ -3005,6 +3009,7 @@ fn emit_map_walk_keys_to_list( k_aver: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let order_idx = registry .map_order_indices_type_idx @@ -3031,6 +3036,7 @@ fn emit_map_walk_keys_to_list( (1, list_ref), ]); // keys = map.keys + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3090,6 +3096,7 @@ fn emit_map_walk_values_to_list( registry: &TypeRegistry, list_idx: u32, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let order_idx = registry .map_order_indices_type_idx @@ -3115,6 +3122,7 @@ fn emit_map_walk_values_to_list( (1, ValType::I32), (1, list_ref), ]); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3170,6 +3178,7 @@ fn emit_map_eq( keyh: KeyHelpers, v_helpers: Option, get_fn_idx: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3200,7 +3209,8 @@ fn emit_map_eq( heap_type: HeapType::Concrete(opt_idx), }); // Locals: 2=typed map_a, 3=typed map_b, 4=cap, 5=i, 6=keys_a, - // 7=values_a, 8=cur_key (boxed), 9=opt result, 10=v_a, 11=v_b + // 7=values_a, 8=cur_key (boxed), 9=opt result, 10=v_a, 11=v_b, + // 12=copy of keys_a, 13=copy of values_a let mut f = Function::new(vec![ (1, map_ref), (1, map_ref), @@ -3212,6 +3222,8 @@ fn emit_map_eq( (1, opt_ref), (1, v_val), (1, v_val), + (1, keys_ref), + (1, values_ref), ]); let map_heap = HeapType::Concrete(slots.map); f.instruction(&Instruction::LocalGet(0)); @@ -3237,6 +3249,7 @@ fn emit_map_eq( f.instruction(&Instruction::Return); f.instruction(&Instruction::End); // cap = a.cap; keys_a = a.keys; values_a = a.values; i = 0 + emit_reroot_local(&mut f, slots.map, reroot_fn, 2); f.instruction(&Instruction::LocalGet(2)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3255,6 +3268,33 @@ fn emit_map_eq( field_index: 3, }); f.instruction(&Instruction::LocalSet(7)); + // Two versions of one lineage share their arrays, and every `get` on + // `b` below makes `b` the current one. Walk a copy of `a`'s buckets + // then, so `a` reads as itself throughout. + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 2, + }); + f.instruction(&Instruction::RefEq); + f.instruction(&Instruction::If(BlockType::Empty)); + for (local, copy, array_type) in [(6, 12, slots.keys_array), (7, 13, slots.values_array)] { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::ArrayNewDefault(array_type)); + f.instruction(&Instruction::LocalTee(copy)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::LocalGet(local)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::ArrayCopy { + array_type_index_dst: array_type, + array_type_index_src: array_type, + }); + f.instruction(&Instruction::LocalGet(copy)); + f.instruction(&Instruction::LocalSet(local)); + } + f.instruction(&Instruction::End); f.instruction(&Instruction::I32Const(0)); f.instruction(&Instruction::LocalSet(5)); // for i in 0..cap @@ -3403,6 +3443,7 @@ fn emit_map_hash( registry: &TypeRegistry, keyh: KeyHelpers, v_helpers: Option, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3434,6 +3475,7 @@ fn emit_map_hash( f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::RefCastNonNull(map_heap)); f.instruction(&Instruction::LocalSet(1)); + emit_reroot_local(&mut f, slots.map, reroot_fn, 1); f.instruction(&Instruction::I32Const(0)); f.instruction(&Instruction::LocalSet(7)); f.instruction(&Instruction::LocalGet(1)); @@ -3506,16 +3548,18 @@ fn emit_map_hash( Ok(f) } -/// `remove(map, k) -> map`. Linear-probe locate the entry; if not +/// `remove(map, k) -> map'`. Linear-probe locate the entry; if not /// found, return the map unchanged. If found, empty its slot and walk /// the rest of the probe run, pulling back every entry that the new /// hole would otherwise hide from its own lookup — the backwards-shift -/// deletion for linear probing. Decrements `map.size`. Same-handle -/// return (mutates in place). +/// deletion for linear probing. Every bucket it writes is a versioned +/// write (`maps/versions.rs`), so `map` keeps its entries and the last +/// version, one entry smaller, is the result. fn emit_map_remove( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3525,7 +3569,7 @@ fn emit_map_remove( // params: 0=map, 1=k. // locals: 2=cap, 3=mask, 4=keys, 5=values, 6=h, 7=i, 8=j, // 9=cur_key, 10=natural, 11=gap, 12=disp, 13=hole, - // 14=hashes, 15=query_hash. + // 14=hashes, 15=query_hash, 16=version, 17=diff, 18=next. // `7=i` is the probe index while the entry is being located, then // the hole the shift is filling — which travels as entries move. // `13=hole` keeps the slot the removed key vacated, unchanged, as @@ -3563,9 +3607,41 @@ fn emit_map_remove( }), ), // 14: hashes (1, ValType::I32), // 15: query_hash + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }), + ), // 16: version + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.diff), + }), + ), // 17: diff + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }), + ), // 18: next version ]); + let versioned = VersionedWrite { + version: 16, + diff: 17, + next: 18, + idx: 7, + cap: 2, + keys: 4, + values: 5, + hashes: 14, + }; // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3675,7 +3751,11 @@ fn emit_map_remove( // there. Those are exactly the entries a stop would strand. // // `hole` (local 13) keeps the slot the removal emptied — the scan's - // wrap guard, see below. The live hole travels in `i`. + // wrap guard, see below. The live hole travels in `i`. `version` + // (local 16) starts at `map` and moves to each new version a write + // makes. + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalSet(16)); f.instruction(&Instruction::LocalGet(7)); f.instruction(&Instruction::LocalSet(13)); f.instruction(&Instruction::LocalGet(7)); @@ -3732,22 +3812,24 @@ fn emit_map_remove( f.instruction(&Instruction::I32GeU); f.instruction(&Instruction::If(BlockType::Empty)); // shift: keys[i] = next; values[i] = values[j]; hashes[i] = hashes[j] - f.instruction(&Instruction::LocalGet(4)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(9)); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(8)); - f.instruction(&Instruction::ArrayGet(slots.values_array)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(8)); - f.instruction(&Instruction::ArrayGet(slots.hashes_array)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); + emit_versioned_write(&mut f, slots, &versioned, |f| { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(9)); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(8)); + f.instruction(&Instruction::ArrayGet(slots.values_array)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(8)); + f.instruction(&Instruction::ArrayGet(slots.hashes_array)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }); // i = j — the slot just vacated is the new hole. Its stale key stays // in the array until the next move overwrites it or the `keys[i] = // null` below clears it, and no lookup can reach it in between. @@ -3770,17 +3852,20 @@ fn emit_map_remove( // box / String / record / carrier / List / Vector concrete idx, // nominal root for sum K). let null_heap = key_storage_null_heap(k_aver, registry); - f.instruction(&Instruction::LocalGet(4)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::RefNull(null_heap)); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); + emit_versioned_write(&mut f, slots, &versioned, |f| { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::RefNull(null_heap)); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }); - // map.size = map.size - 1 - f.instruction(&Instruction::LocalGet(0)); + // The last version is new and nobody else holds it yet: + // version.size = map.size - 1 + f.instruction(&Instruction::LocalGet(16)); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3793,8 +3878,8 @@ fn emit_map_remove( field_index: 0, }); - // return map - f.instruction(&Instruction::LocalGet(0)); + // return the last version + f.instruction(&Instruction::LocalGet(16)); f.instruction(&Instruction::End); Ok(f) } @@ -3804,6 +3889,7 @@ fn emit_map_entries( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3849,6 +3935,7 @@ fn emit_map_entries( (1, ValType::I32), (1, lt_ref), ]); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3910,17 +3997,10 @@ fn emit_map_entries( /// `from_list(l) -> Map`. Walks `l` from head to tail, /// struct.get's the (K, V) from each tuple, calls the per-(K, V) -/// `set_in_place` helper to insert. Allocates a fresh empty map (via -/// the per-(K, V) `empty` shape inlined: cap = INITIAL_CAP, fresh keys -/// and values arrays) and returns it. -/// -/// In place rather than clone-on-write because the map is allocated -/// here, is held only by this frame's local, and is not handed to -/// anyone until the walk finishes — the uniqueness the alias pass -/// proves at a `Map.set` call site is a fact of this body's shape. -/// Copying instead would make `Map.fromList` quadratic: one full array -/// copy per pair. With growth in the insert helper that would be the -/// only remaining size cliff a map has. +/// `set` helper to insert. Allocates a fresh empty map (via the +/// per-(K, V) `empty` shape inlined: cap = INITIAL_CAP, fresh keys +/// and values arrays) and returns it. `set` writes in place, so the +/// walk is linear in the list. fn emit_map_from_list( canonical: &str, registry: &TypeRegistry, @@ -3969,6 +4049,7 @@ fn emit_map_from_list( f.instruction(&Instruction::ArrayNewDefault(slots.values_array)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(slots.hashes_array)); + emit_no_diff(&mut f, slots); f.instruction(&Instruction::StructNew(slots.map)); f.instruction(&Instruction::LocalSet(2)); // cur = l diff --git a/src/codegen/wasm_gc/maps/versions.rs b/src/codegen/wasm_gc/maps/versions.rs new file mode 100644 index 000000000..a997d8bd2 --- /dev/null +++ b/src/codegen/wasm_gc/maps/versions.rs @@ -0,0 +1,335 @@ +//! Versions of one map that share its arrays. +//! +//! A `Map` value is a `$map` struct over three arrays. `Map.set` and +//! `Map.remove` write into those arrays in place and return a new `$map` +//! struct over the same arrays, so an update costs a probe, not a copy of +//! the table. The map it was given stays a valid value: before the write, +//! the helper records what the bucket held in a `$diff` struct and hangs it +//! on the old `$map` (`diff` field). The old version is then "the new +//! version, except that bucket `idx` holds `key`, `value`, `hash`". +//! +//! Exactly one version of a lineage owns the arrays' current contents: the +//! one whose `diff` is null. [`emit_map_reroot`] makes a given version that +//! one. It follows the `diff` chain to the current version and walks back, +//! swapping each recorded bucket into the arrays and hanging the swapped-out +//! contents on the version it just left, so every version stays readable +//! and the chain now points the other way. Every helper that reads or writes +//! a map's arrays reroots first; `len` reads only the version's own `size`. +//! +//! A program that uses each map once, as the generated loop and most +//! recursive builders do, never has more than one diff to undo, so an update +//! is O(1). A program that keeps an old version and reads it again pays for +//! the buckets written since, once, and the version it moved away from pays +//! the same to come back. A grow allocates new arrays and leaves the old +//! ones to the old version, so no diff crosses a grow. +//! +//! This is the persistent-array technique of Baker ("Shallow binding makes +//! functional arrays fast", 1991), as in Conchon and Filliâtre's +//! "A Persistent Union-Find Data Structure" (2007). +//! +//! Two versions of one lineage cannot both be current. A helper that reads +//! two maps at once (`eq`) therefore copies one side when both share arrays. +//! The host reads a map's buckets directly only for the sockets of +//! `Tcp.poll` and the wait set of `Wait.poll`; those arguments are rerooted +//! before the import is called. + +use wasm_encoder::{BlockType, Function, HeapType, Instruction, RefType, ValType}; + +use super::super::WasmGcError; +use super::super::types::{MapSlots, TypeRegistry}; +use super::key_storage_val_type; + +/// `$map` field holding the version's diff; null on the current version. +const DIFF_FIELD: u32 = 5; + +const DIFF_NEXT: u32 = 0; +const DIFF_IDX: u32 = 1; +const DIFF_KEY: u32 = 2; +const DIFF_VALUE: u32 = 3; +const DIFF_HASH: u32 = 4; + +fn nullable(idx: u32) -> ValType { + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(idx), + }) +} + +/// Push a null diff: the operand every `struct.new $map` of a current +/// version ends with. +pub(in crate::codegen::wasm_gc) fn emit_no_diff(f: &mut Function, slots: MapSlots) { + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.diff))); +} + +/// Make the version in `map_local` the current one before its arrays are +/// read. The null-diff test is inline, so a map that is already current, +/// which is nearly every map, pays a field read and no call. +pub(in crate::codegen::wasm_gc) fn emit_reroot_local( + f: &mut Function, + map_type: u32, + reroot_fn: u32, + map_local: u32, +) { + f.instruction(&Instruction::LocalGet(map_local)); + f.instruction(&Instruction::StructGet { + struct_type_index: map_type, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::I32Eqz); + f.instruction(&Instruction::If(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(map_local)); + f.instruction(&Instruction::Call(reroot_fn)); + f.instruction(&Instruction::LocalSet(map_local)); + f.instruction(&Instruction::End); +} + +/// `reroot(map) -> map`: make `map` the version that owns its arrays' +/// contents, and return it. +pub(super) fn emit_map_reroot( + canonical: &str, + registry: &TypeRegistry, + slots: MapSlots, +) -> Result { + let (k_aver, v_aver) = super::super::types::parse_map_kv(canonical).ok_or_else(|| { + WasmGcError::Validation(format!("Map.reroot: bad canonical `{canonical}`")) + })?; + let v_val = super::super::types::aver_to_wasm(v_aver, Some(registry))?.ok_or_else(|| { + WasmGcError::Validation(format!( + "Map value type `{v_aver}` has no wasm representation" + )) + })?; + let map_ref = nullable(slots.map); + let diff_ref = nullable(slots.diff); + // params: 0=map. locals: 1=prev, 2=cur, 3=diff, 4=next, 5=current + // version, 6=version being restored, 7=idx, 8=keys, 9=values, + // 10=hashes, 11/12/13 = the bucket's key/value/hash being swapped out. + let mut f = Function::new([ + (1, map_ref), // 1: prev + (1, map_ref), // 2: cur + (1, diff_ref), // 3: diff + (1, map_ref), // 4: next + (1, map_ref), // 5: current version + (1, map_ref), // 6: version being restored + (1, ValType::I32), // 7: idx + (1, nullable(slots.keys_array)), // 8: keys + (1, nullable(slots.values_array)), // 9: values + (1, nullable(slots.hashes_array)), // 10: hashes + (1, key_storage_val_type(k_aver, registry)?), // 11: key out + (1, v_val), // 12: value out + (1, ValType::I32), // 13: hash out + ]); + let get_map = |f: &mut Function, field| { + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: field, + }); + }; + let get_diff = |f: &mut Function, field| { + f.instruction(&Instruction::StructGet { + struct_type_index: slots.diff, + field_index: field, + }); + }; + let set_diff = |f: &mut Function, field| { + f.instruction(&Instruction::StructSet { + struct_type_index: slots.diff, + field_index: field, + }); + }; + + // Already current: nothing to do. This is every call on a map used once. + f.instruction(&Instruction::LocalGet(0)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::If(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::Return); + f.instruction(&Instruction::End); + + // Walk to the current version, turning each diff's `next` round to + // point at the version before it. `prev` ends on the last stale + // version, whose diff is relative to the current one. + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.map))); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalSet(2)); + f.instruction(&Instruction::Block(BlockType::Empty)); + f.instruction(&Instruction::Loop(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(2)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::LocalTee(3)); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::BrIf(1)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalSet(4)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(1)); + set_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalSet(2)); + f.instruction(&Instruction::Br(0)); + f.instruction(&Instruction::End); + f.instruction(&Instruction::End); + + // Every version in the chain shares the current version's arrays. + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::LocalSet(5)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 2); + f.instruction(&Instruction::LocalSet(8)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 3); + f.instruction(&Instruction::LocalSet(9)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 4); + f.instruction(&Instruction::LocalSet(10)); + + // Walk back towards `map`, one version at a time: put that version's + // bucket back into the arrays, and keep what was there as the diff of + // the version just left, pointing forward to the restored one. + f.instruction(&Instruction::Block(BlockType::Empty)); + f.instruction(&Instruction::Loop(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(1)); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::BrIf(1)); + f.instruction(&Instruction::LocalGet(1)); + f.instruction(&Instruction::LocalSet(6)); + f.instruction(&Instruction::LocalGet(6)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::LocalSet(3)); + // prev = the version before this one (the reversed link). + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_IDX); + f.instruction(&Instruction::LocalSet(7)); + // Swap the bucket out. + for (array_local, array_type, out_local) in [ + (8, slots.keys_array, 11), + (9, slots.values_array, 12), + (10, slots.hashes_array, 13), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::ArrayGet(array_type)); + f.instruction(&Instruction::LocalSet(out_local)); + } + // Put the restored version's bucket in. + for (array_local, array_type, field) in [ + (8, slots.keys_array, DIFF_KEY), + (9, slots.values_array, DIFF_VALUE), + (10, slots.hashes_array, DIFF_HASH), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, field); + f.instruction(&Instruction::ArraySet(array_type)); + } + // The diff now describes the version just left, relative to the + // restored one. + for (out_local, field) in [(11, DIFF_KEY), (12, DIFF_VALUE), (13, DIFF_HASH)] { + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(out_local)); + set_diff(&mut f, field); + } + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(6)); + set_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(6)); + emit_no_diff(&mut f, slots); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalSet(5)); + f.instruction(&Instruction::Br(0)); + f.instruction(&Instruction::End); + f.instruction(&Instruction::End); + + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::End); + Ok(f) +} + +/// Locals a versioned write reads and writes. `version` holds the current +/// version before the write and the new one after it. +pub(super) struct VersionedWrite { + pub version: u32, + pub diff: u32, + pub next: u32, + pub idx: u32, + pub cap: u32, + pub keys: u32, + pub values: u32, + pub hashes: u32, +} + +/// Write one bucket of the current version in `w.version` as a new version. +/// +/// Records the bucket's key, value and hash on the old version, lets +/// `write` store the new ones (it runs with nothing on the stack and must +/// leave nothing), and moves `w.version` to a new `$map` over the same +/// arrays with the old version's size. The caller adjusts the size of the +/// last version it makes. +pub(super) fn emit_versioned_write( + f: &mut Function, + slots: MapSlots, + w: &VersionedWrite, + write: impl FnOnce(&mut Function), +) { + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.map))); + f.instruction(&Instruction::LocalGet(w.idx)); + for (array_local, array_type) in [ + (w.keys, slots.keys_array), + (w.values, slots.values_array), + (w.hashes, slots.hashes_array), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(w.idx)); + f.instruction(&Instruction::ArrayGet(array_type)); + } + f.instruction(&Instruction::StructNew(slots.diff)); + f.instruction(&Instruction::LocalSet(w.diff)); + + write(f); + + f.instruction(&Instruction::LocalGet(w.version)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + f.instruction(&Instruction::LocalGet(w.cap)); + f.instruction(&Instruction::LocalGet(w.keys)); + f.instruction(&Instruction::LocalGet(w.values)); + f.instruction(&Instruction::LocalGet(w.hashes)); + emit_no_diff(f, slots); + f.instruction(&Instruction::StructNew(slots.map)); + f.instruction(&Instruction::LocalSet(w.next)); + f.instruction(&Instruction::LocalGet(w.diff)); + f.instruction(&Instruction::LocalGet(w.next)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.diff, + field_index: DIFF_NEXT, + }); + f.instruction(&Instruction::LocalGet(w.version)); + f.instruction(&Instruction::LocalGet(w.diff)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(w.next)); + f.instruction(&Instruction::LocalSet(w.version)); +} diff --git a/src/codegen/wasm_gc/module.rs b/src/codegen/wasm_gc/module.rs index cdfdf7399..c10bb5ecb 100644 --- a/src/codegen/wasm_gc/module.rs +++ b/src/codegen/wasm_gc/module.rs @@ -2198,6 +2198,15 @@ pub(super) fn emit_module_with( headers_values_array_type_idx: map_slots.values_array, headers_hashes_array_type_idx: map_slots.hashes_array, headers_map_type_idx: map_slots.map, + headers_map_diff_type_idx: map_slots.diff, + headers_map_reroot_fn: map_helpers + .kv_helpers("Map>") + .ok_or_else(|| { + WasmGcError::Validation( + "Http headers need the Map> helpers".into(), + ) + })? + .reroot, list_string_type_idx: list_string_idx, option_list_string_type_idx: opt_list_string_idx, aint_from_i64_fn_idx: registry.aint_from_i64_fn_idx, @@ -3645,6 +3654,15 @@ pub(super) fn emit_module_with( headers_values_array_type_idx: map_slots.values_array, headers_hashes_array_type_idx: map_slots.hashes_array, headers_map_type_idx: map_slots.map, + headers_map_diff_type_idx: map_slots.diff, + headers_map_reroot_fn: map_helpers + .kv_helpers("Map>") + .ok_or_else(|| { + WasmGcError::Validation( + "Http headers need the Map> helpers".into(), + ) + })? + .reroot, list_string_type_idx: list_string_idx, option_list_string_type_idx: opt_list_string_idx, aint_to_i64_checked_fn_idx: registry.aint_to_i64_checked_fn_idx, @@ -6859,6 +6877,48 @@ fn emit_user_types( element_type: wasm_encoder::StorageType::Val(hashes_ref), mutable: true, }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::Ref( + wasm_encoder::RefType { + nullable: true, + heap_type: wasm_encoder::HeapType::Concrete(slots.diff), + }, + )), + mutable: true, + }, + ]), + )); + // One bucket of an older version: the version it leads to, the + // bucket, and the key, value and hash that bucket held before the + // newer version wrote over it (see `maps.rs`). + entries.push(( + slots.diff, + mk_struct(vec![ + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::Ref( + wasm_encoder::RefType { + nullable: true, + heap_type: wasm_encoder::HeapType::Concrete(slots.map), + }, + )), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::I32), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(key_storage_val), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(v_val), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::I32), + mutable: true, + }, ]), )); } @@ -10701,6 +10761,7 @@ fn emit_factory_map_string_list_string_empty( f.instruction(&Instruction::RefNull(wasm_encoder::HeapType::Concrete( slots.hashes_array, ))); + super::maps::emit_no_diff(&mut f, slots); f.instruction(&Instruction::StructNew(slots.map)); f.instruction(&Instruction::End); Ok(f) @@ -10952,6 +11013,13 @@ fn emit_handler_wrapper( field_index: 2, }); f.instruction(&Instruction::LocalSet(9)); + let headers_reroot = fn_map + .map_helpers_lookup("Map>") + .ok_or(WasmGcError::Validation( + "aver_http_handle wrapper requires the Map> helpers".into(), + ))? + .reroot; + super::maps::emit_reroot_local(&mut f, map_slots.map, headers_reroot, 9); // Read map cap + arrays into iteration slots. f.instruction(&Instruction::LocalGet(9)); diff --git a/src/codegen/wasm_gc/types.rs b/src/codegen/wasm_gc/types.rs index 44e654103..4ffdb074d 100644 --- a/src/codegen/wasm_gc/types.rs +++ b/src/codegen/wasm_gc/types.rs @@ -348,8 +348,13 @@ pub(super) struct MapSlots { /// `(array (mut i32))` — cached complete key hash per occupied bucket. pub(super) hashes_array: u32, /// `(struct (mut i32 size) (mut i32 cap) (mut keys_ref) (mut values_ref) - /// (mut hashes_ref))`. + /// (mut hashes_ref) (mut diff_ref))`. A null `diff` marks the + /// version that owns the arrays' current contents; see `maps.rs`. pub(super) map: u32, + /// `(struct (mut map_ref next) (mut i32 idx) (mut K key) (mut V value) + /// (mut i32 hash))` — what one bucket held in an older version + /// of a map whose arrays a newer version has since written to. + pub(super) diff: u32, } impl TypeRegistry { @@ -1103,7 +1108,8 @@ impl TypeRegistry { next_idx += 1; } } - // Allocate four slots: keys_array, values_array, hashes_array, map. + // Allocate five slots: keys_array, values_array, hashes_array, map, + // diff. // Order: arrays first so the struct (higher idx) can // reference them without crossing rec-group boundaries. let keys_array = next_idx; @@ -1114,6 +1120,8 @@ impl TypeRegistry { next_idx += 1; let map = next_idx; next_idx += 1; + let diff = next_idx; + next_idx += 1; map_types.insert( canonical.clone(), MapSlots { @@ -1121,6 +1129,7 @@ impl TypeRegistry { values_array, hashes_array, map, + diff, }, ); map_order.push(canonical); diff --git a/src/codegen/wasm_gc/wasip2_http.rs b/src/codegen/wasm_gc/wasip2_http.rs index c59d50ad0..9ce44ddb2 100644 --- a/src/codegen/wasm_gc/wasip2_http.rs +++ b/src/codegen/wasm_gc/wasip2_http.rs @@ -72,6 +72,10 @@ pub(super) struct HttpGetIndices { pub headers_values_array_type_idx: u32, pub headers_hashes_array_type_idx: u32, pub headers_map_type_idx: u32, + /// The map's `$diff` struct: a fresh map carries a null one. + pub headers_map_diff_type_idx: u32, + /// The map's `reroot` helper, called before its buckets are read. + pub headers_map_reroot_fn: u32, /// `List` cons-cell type idx. Each header value lands /// either in a singleton `[value]` list or prepended onto the /// existing list when the same field-key reappears (Set-Cookie @@ -773,6 +777,12 @@ pub(super) fn emit_http_get(indices: &HttpGetIndices, h: &HttpGetHelperFns) -> F // but the dispatcher passes an empty map, so it's a no-op // beyond the cap-iter (~16k iterations of "is keys[i] null? // yes, skip"). Acceptable for v1 PoC. + super::maps::emit_reroot_local( + &mut f, + indices.headers_map_type_idx, + indices.headers_map_reroot_fn, + p_headers, + ); f.instruction(&Instruction::LocalGet(p_headers)); f.instruction(&Instruction::StructGet { struct_type_index: indices.headers_map_type_idx, @@ -1523,6 +1533,9 @@ pub(super) fn emit_http_get(indices: &HttpGetIndices, h: &HttpGetHelperFns) -> F f.instruction(&Instruction::ArrayNewDefault(values_arr_idx)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(hashes_arr_idx)); + f.instruction(&Instruction::RefNull(HeapType::Concrete( + indices.headers_map_diff_type_idx, + ))); f.instruction(&Instruction::StructNew(map_idx)); f.instruction(&Instruction::LocalSet(l_h_map)); diff --git a/src/codegen/wasm_gc/wasip2_http_handler.rs b/src/codegen/wasm_gc/wasip2_http_handler.rs index da77ab06a..f890703e8 100644 --- a/src/codegen/wasm_gc/wasip2_http_handler.rs +++ b/src/codegen/wasm_gc/wasip2_http_handler.rs @@ -99,6 +99,10 @@ pub(super) struct ServerHandlerIndices { pub headers_values_array_type_idx: u32, pub headers_hashes_array_type_idx: u32, pub headers_map_type_idx: u32, + /// The map's `$diff` struct: a fresh map carries a null one. + pub headers_map_diff_type_idx: u32, + /// The map's `reroot` helper, called before its buckets are read. + pub headers_map_reroot_fn: u32, /// `List` cons-cell type idx — head = String ref, tail /// = list ref. pub list_string_type_idx: u32, @@ -699,6 +703,9 @@ pub(super) fn emit_aver_http_handle( f.instruction(&Instruction::ArrayNewDefault(values_arr_idx)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(hashes_arr_idx)); + f.instruction(&Instruction::RefNull(HeapType::Concrete( + indices.headers_map_diff_type_idx, + ))); f.instruction(&Instruction::StructNew(map_idx)); f.instruction(&Instruction::LocalSet(l_req_headers_map)); @@ -1167,6 +1174,12 @@ pub(super) fn emit_aver_http_handle( } // Walk response headers map → fields.append. + super::maps::emit_reroot_local( + &mut f, + map_idx, + indices.headers_map_reroot_fn, + l_resp_headers_map, + ); f.instruction(&Instruction::LocalGet(l_resp_headers_map)); f.instruction(&Instruction::StructGet { struct_type_index: map_idx, diff --git a/tests/wasm_gc_map_versions_spec.rs b/tests/wasm_gc_map_versions_spec.rs new file mode 100644 index 000000000..2cc6d4de1 --- /dev/null +++ b/tests/wasm_gc_map_versions_spec.rs @@ -0,0 +1,224 @@ +//! Versions of one wasm-gc map. +//! +//! `Map.set` and `Map.remove` write into the map's arrays in place and +//! leave the map they were given valid through a diff (see +//! `src/codegen/wasm_gc/maps/versions.rs`). These tests keep every older +//! version in use after newer ones are made from it: across updates, +//! inserts, removes that shift a probe run, growth, a long chain of +//! overwrites, two branches from one base, and equality between versions +//! that share arrays. Each program runs on the VM and on wasm-gc, and both +//! must print the hand-checked answer. +//! +//! Before versions, `Map.remove` wrote into the map it was given, so a +//! caller that kept that map saw the key disappear, and `Map.set` copied +//! the whole table on every call whose receiver it could not prove unique +//! (an answer module's state field, for one). + +#![cfg(feature = "wasm")] + +#[path = "support/aver_cmd.rs"] +mod aver_cmd; + +use aver_cmd::{cleanup, format_output, temp_module}; + +use std::path::PathBuf; +use std::process::Command; + +fn run_cli(prefix: &str, source: &str, extra_args: &[&str]) -> String { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = temp_module(prefix, source); + let out = Command::new(env!("CARGO_BIN_EXE_aver")) + .current_dir(&repo_root) + .arg("run") + .arg(&path) + .args(extra_args) + .output() + .expect("expected `aver run` to execute"); + cleanup(&path); + assert!( + out.status.success(), + "{prefix} run {extra_args:?} failed:\n{}", + format_output(&out) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn assert_vm_and_wasm_gc(name: &str, source: &str, expected: &str) { + let vm = run_cli(name, source, &[]); + assert_eq!( + vm, expected, + "{name}: the VM diverged from the checked answer" + ); + let wasm = run_cli(name, source, &["--wasm-gc"]); + assert_eq!( + wasm, expected, + "{name}: wasm-gc diverged from the checked answer — a version of a map \ + saw a write made to a later one" + ); +} + +#[test] +fn remove_leaves_the_map_it_was_given_alone() { + assert_vm_and_wasm_gc( + "map-versions-remove", + r#" +module Main + intent = "Remove and set from one map, then read all three." + effects [Console.print] + +fn both(m: Map) -> Tuple, Map> + ? "Two maps from one." + (Map.remove(m, 1), Map.set(m, 2, 20)) + +fn main() -> Unit + ! [Console.print] + base = Map.set(Map.set({}, 1, 10), 3, 30) + pair = both(base) + match pair + (a, b) -> Console.print("{Map.len(base)} {Map.has(base, 1)} {Map.len(a)} {Map.has(a, 1)} {Map.len(b)} {Map.has(b, 1)} {Map.has(b, 2)} {Map.has(base, 2)}") +"#, + "2 true 1 false 3 true true false", + ); +} + +#[test] +fn every_version_reads_as_itself() { + assert_vm_and_wasm_gc( + "map-versions-all", + r#" +module Main + intent = "Every version of a map stays what it was when the next one is made from it." + effects [Console.print] + +fn fill(m: Map, from: Int, to: Int) -> Map + ? "Maps each key in from..to-1 to ten times itself." + match from >= to + true -> m + false -> fill(Map.set(m, from, from * 10), from + 1, to) + +fn total(m: Map, keys: List, acc: Int) -> Int + ? "Sums key * 1000 + value over the listed keys, and counts an absent one as -1." + match keys + [] -> acc + [k, ..rest] -> match Map.get(m, k) + Option.Some(v) -> total(m, rest, acc + k * 1000 + v) + Option.None -> total(m, rest, acc - 1) + +fn summary(m: Map) -> String + ? "Size, a checksum of the entries, and the first keys in order." + "{Map.len(m)}:{total(m, Map.keys(m), 0)}:{firstKeys(List.take(Map.keys(m), 4), "")}" + +fn firstKeys(keys: List, acc: String) -> String + ? "The keys, comma-separated." + match keys + [] -> acc + [k, ..rest] -> firstKeys(rest, "{acc},{k}") + +fn chain(m: Map, left: Int) -> Map + ? "Overwrites keys left down to 1 with 7." + match left <= 0 + true -> m + false -> chain(Map.set(m, left, 7), left - 1) + +fn removeAll(m: Map, keys: List) -> Map + ? "Removes every listed key." + match keys + [] -> m + [k, ..rest] -> removeAll(Map.remove(m, k), rest) + +fn main() -> Unit + ! [Console.print] + base = fill({}, 0, 40) + grown = fill(base, 40, 100) + updated = Map.set(base, 3, 999) + other = Map.set(base, 3, 111) + removed = removeAll(base, [0, 16, 32, 5, 21, 37, 99]) + long = chain(base, 30) + Console.print(summary(base)) + Console.print(summary(grown)) + Console.print(summary(updated)) + Console.print(summary(other)) + Console.print(summary(removed)) + Console.print(summary(long)) + Console.print(summary(base)) + Console.print("{base == fill({}, 0, 40)} {updated == other} {Map.set(base, 3, 30) == base} {removed == base} {Map.has(removed, 16)} {Map.has(base, 16)}") +"#, + "40:787800:,0,1,2,3\n\ + 100:4999500:,0,1,2,3\n\ + 40:788769:,0,1,2,3\n\ + 40:787881:,0,1,2,3\n\ + 34:675690:,1,2,3,4\n\ + 40:783360:,0,1,2,3\n\ + 40:787800:,0,1,2,3\n\ + true false true false false true", + ); +} + +/// The printed WAT of one `aver compile --target wasm-gc` of `source`. +fn wat_of(source: &str) -> String { + let mut items = aver::source::parse_source(source).expect("parse"); + let neutral_policy = aver::ir::NeutralAllocPolicy; + let result = aver::ir::pipeline::run( + &mut items, + aver::ir::PipelineConfig { + typecheck: Some(aver::ir::TypecheckMode::Full { base_dir: None }), + alloc_policy: Some(&neutral_policy), + run_interp_lower: false, + run_buffer_build: false, + run_chars_fusion: false, + run_list_build: false, + ..Default::default() + }, + ); + let tc = result.typecheck.as_ref().expect("typecheck requested"); + assert!(tc.errors.is_empty(), "typecheck failed: {:?}", tc.errors); + let bytes = aver::codegen::wasm_gc::compile_to_wasm_gc(&items, result.analysis.as_ref()) + .expect("wasm-gc compile"); + wasmprinter::print_bytes(&bytes).expect("print wat") +} + +/// An answer module's state reaches `Map.set` as a field of a record the +/// function was handed, which no ownership fact covers. The insert must +/// still not copy the table: its only array allocations are the grow's, and +/// it never copies an array. +#[test] +fn set_on_a_record_field_does_not_copy_the_table() { + let wat = wat_of( + r#" +module Main + intent = "A state record whose map every request updates." + effects [Console.print] + +record State + counts: Map + +fn bump(state: State, key: Int) -> State + ? "One more for this key." + next = Option.withDefault(Map.get(state.counts, key), 0) + 1 + State(counts = Map.set(state.counts, key, next)) + +verify bump + bump(State(counts = {}), 7) => State(counts = {7 => 1}) + +fn main() -> Unit + ! [Console.print] + Console.print("{Map.len(bump(State(counts = {}), 1).counts)}") +"#, + ); + let start = wat + .find("(func $\"Map.set Map") + .unwrap_or_else(|| panic!("no named Map.set helper in:\n{wat}")); + let body = &wat[start..]; + let body = &body[..body[1..] + .find("\n (func") + .map_or(body.len(), |end| end + 1)]; + assert!( + !body.contains("array.copy"), + "Map.set copies an array:\n{body}" + ); + assert_eq!( + body.matches("array.new_default").count(), + 3, + "Map.set allocates arrays outside its grow:\n{body}" + ); +}