From 05988a6f4524448d52debfb2ae215caa18198278 Mon Sep 17 00:00:00 2001 From: jasisz Date: Fri, 25 Sep 2026 03:58:57 +0200 Subject: [PATCH 1/4] wasm-gc: update maps in place and keep older versions valid Map.set copied the whole table whenever the compiler could not prove the map had no other holder, which it cannot for a map held in a record field. An answer module's state map of 100 000 entries took 558 s to serve 2000 Map.set requests under `aver run --wasm-gc`. Map.remove wrote into the map it was given with no check at all, so a caller that kept that map saw the key disappear. A map now keeps versions over shared arrays. set and remove write the bucket in place and return a new map struct. Before each write, the old contents of the bucket are recorded in a diff struct on the map they were given. The version with no diff owns the arrays' contents. A new per-map reroot helper, in the slot set_in_place used to have, makes any version the current one by swapping recorded buckets back. Every helper that reads the arrays reroots first. So does the code outside maps.rs that walks buckets: the wasip2 header walkers, the http handler wrapper, and the map arguments of host imports and of the wasip2 poll. eq copies one side when both maps share arrays. A grow uses new arrays, so no diff crosses one. The same run now takes 7 s, nearly all of it startup and building the map, and 0.3 s on Node 26 instead of 1.9 s. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + src/codegen/wasm_gc/body/builtins_wasip2.rs | 4 + src/codegen/wasm_gc/body/from_mir/builtins.rs | 37 +- src/codegen/wasm_gc/body/from_mir/mod.rs | 5 + src/codegen/wasm_gc/maps.rs | 611 ++++++++++-------- src/codegen/wasm_gc/maps/versions.rs | 324 ++++++++++ src/codegen/wasm_gc/module.rs | 68 ++ src/codegen/wasm_gc/types.rs | 13 +- src/codegen/wasm_gc/wasip2_http.rs | 8 + src/codegen/wasm_gc/wasip2_http_handler.rs | 8 + tests/wasm_gc_map_versions_spec.rs | 224 +++++++ 11 files changed, 1026 insertions(+), 277 deletions(-) create mode 100644 src/codegen/wasm_gc/maps/versions.rs create mode 100644 tests/wasm_gc_map_versions_spec.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e21def1d..44b25284c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The generated loop is now written from the program's source alone, and the manif ### Fixed +- **wasm-gc: `Map.set` no longer copies the map, and `Map.remove` no longer changes the map it was given.** `set` copied every bucket unless the compiler could prove the map had no other holder, which it cannot for a map held in a record field such as an answer module's state. A 100 000-entry state map served 2000 `Map.set` requests in 558 s under `aver run --wasm-gc` and 1.9 s on Node 26. `remove` wrote into the map it was given, so a caller that still held that map saw the key gone. Both now write into the map's arrays in place and return a new version. The version they were given stays valid, because a record of what the write replaced is kept with it. The same run now takes 7 s under `aver run --wasm-gc`, nearly all of it startup and building the map, and 0.3 s on Node. Reading an older version again costs one step for each write made since. - **A wait over sockets and jobs keeps watching its sockets after a job outside its set settles.** The wake from that job used to end the socket poll, and the wait then slept out the rest of its timeout on the job engine alone, missing sockets that became ready meanwhile and never reporting them. The VM, generated Rust and the wasm-gc native host now share one wait loop that polls the whole set again. - **A handle whose slot the engine has forgotten answers `work: unknown job` on the VM**, as it already did elsewhere, instead of claiming another job kind started it. Which kind began a job is now kept in the job's own slot, so nothing a job kind keeps grows with the number of jobs it starts. - **The VM runs a function whose bytecode is larger than 32 KiB.** Jump offsets were sixteen bits and a longer forward jump wrapped into a backward one, which crashed `aver verify` on large generated trace laws. diff --git a/src/codegen/wasm_gc/body/builtins_wasip2.rs b/src/codegen/wasm_gc/body/builtins_wasip2.rs index 267dfa0d8..67e9748a4 100644 --- a/src/codegen/wasm_gc/body/builtins_wasip2.rs +++ b/src/codegen/wasm_gc/body/builtins_wasip2.rs @@ -1004,6 +1004,9 @@ fn emit_poll_wasip2( WasmGcError::Validation(format!("{operation} on wasip2: helper fn idx missing")) })?; emit_mir_expr(func, &args[0], slots, ctx)?; + // The poll helper walks the map's buckets directly: hand it the + // current version. + super::from_mir::emit_map_arg_current(func, &args[0], ctx)?; emit_mir_expr(func, &args[1], slots, ctx)?; func.instruction(&Instruction::Call(helper)); Ok(()) @@ -1750,6 +1753,7 @@ fn emit_http_simple_method_wasip2( func.instruction(&Instruction::ArrayNewDefault(map_slots.values_array)); func.instruction(&Instruction::I32Const(INITIAL_CAP)); func.instruction(&Instruction::ArrayNewDefault(map_slots.hashes_array)); + crate::codegen::wasm_gc::maps::emit_no_diff(func, map_slots); func.instruction(&Instruction::StructNew(map_slots.map)); func.instruction(&Instruction::Call(fn_idx)); Ok(()) diff --git a/src/codegen/wasm_gc/body/from_mir/builtins.rs b/src/codegen/wasm_gc/body/from_mir/builtins.rs index f597acb0e..98bc9402c 100644 --- a/src/codegen/wasm_gc/body/from_mir/builtins.rs +++ b/src/codegen/wasm_gc/body/from_mir/builtins.rs @@ -1966,11 +1966,34 @@ pub(crate) fn emit_mir_result_from_option( Ok(MirBuiltinEmit::Produced(true)) } +/// With a `Map` argument of `arg`'s type on the stack, make it the current +/// version of its map (`maps/versions.rs`). A host import reads the map's +/// buckets directly and cannot do it itself. Any other type is left alone. +pub(crate) fn emit_map_arg_current( + func: &mut Function, + arg: &Spanned, + ctx: &EmitCtx<'_>, +) -> Result<(), WasmGcError> { + let aver = aver_type_str_of(arg); + let canonical: String = aver.chars().filter(|c| !c.is_whitespace()).collect(); + if !canonical.starts_with("Map<") { + return Ok(()); + } + let helpers = ctx + .fn_map + .map_helpers_lookup(&canonical) + .ok_or(WasmGcError::Validation(format!( + "a `{aver}` argument crosses to the host but no Map helpers are registered for it" + )))?; + func.instruction(&Instruction::Call(helpers.reroot)); + Ok(()) +} + /// Mirror of `emit_map_kv_call`: the `Map.*` methods dispatch to the /// per-`Map` helpers (`fn_map.map_helpers_lookup`). `has` reuses -/// the `get_pair` helper and drops the value; `set` picks `set_in_place` -/// vs the clone-on-write `set` by `mir_arg_uniquely_owned` (the MIR -/// analogue of the oracle's `arg_uniquely_owned`). The canonical comes +/// the `get_pair` helper and drops the value. `set` and `remove` need no +/// ownership fact: they write in place and leave the map they were given +/// valid as an older version (`maps/versions.rs`). The canonical comes /// from the map arg's stamped type; every arg recurses `emit_mir_expr`. pub(crate) fn emit_mir_map_builtin( func: &mut Function, @@ -2062,13 +2085,7 @@ pub(crate) fn emit_mir_map_builtin( "Map.{method}: map argument has type `{map_aver}` but no helpers are registered" )))?; match method { - "set" => { - if mir_arg_uniquely_owned(&args[0], ctx) { - helpers.set_in_place - } else { - helpers.set - } - } + "set" => helpers.set, "get" => helpers.get, "len" => helpers.len, "keys" => helpers.keys, diff --git a/src/codegen/wasm_gc/body/from_mir/mod.rs b/src/codegen/wasm_gc/body/from_mir/mod.rs index df9612c5e..773d89458 100644 --- a/src/codegen/wasm_gc/body/from_mir/mod.rs +++ b/src/codegen/wasm_gc/body/from_mir/mod.rs @@ -84,6 +84,7 @@ pub(super) use super::slots::count_value_params; pub(super) use super::{CallerFnCollector, EmitCtx, FnMap, SlotTable, Wasip2Lowering}; mod builtins; +pub(super) use builtins::emit_map_arg_current; mod collections; mod constructors; mod control; @@ -914,6 +915,10 @@ pub(crate) fn emit_mir_expr( if emit_mir_expr(func, arg, slots, ctx)?.is_none() { return Ok(None); } + // The host reads a map's buckets directly + // (`Tcp.poll`, `Wait.poll`), so hand it the + // current version. + builtins::emit_map_arg_current(func, arg, ctx)?; if ctx.registry.bignum && int_args.contains(&i) { // CHECKED (not saturating): an out-of-i64 Int // crossing the host effect boundary must diff --git a/src/codegen/wasm_gc/maps.rs b/src/codegen/wasm_gc/maps.rs index 28797a66e..040a82f96 100644 --- a/src/codegen/wasm_gc/maps.rs +++ b/src/codegen/wasm_gc/maps.rs @@ -16,18 +16,25 @@ //! mut (ref null $keys_array) keys; //! mut (ref null $values_array) values; //! mut (ref null $hashes_array) hashes; +//! mut (ref null $diff_KV) diff; //! } //! ``` //! +//! `set` and `remove` write into the arrays in place and return a new +//! struct over them; the map they were given stays a valid value through +//! the `diff` the write leaves on it. Several versions of one map share +//! its arrays, and a helper makes the version it was handed current +//! (`reroot`) before it reads them. See `maps/versions.rs`. +//! //! Empty slot marker = `keys[i] == null` (only valid for `K` that //! cannot legitimately be null, which Aver guarantees for ref types //! since the type system rejects null at the source level). //! -//! The table grows. Both insert helpers check, before they probe, -//! whether the entry they are about to add would push occupancy past +//! The table grows. The insert helper checks, before it probes, +//! whether the entry it is about to add would push occupancy past //! three quarters of capacity (see [`LOAD_SHIFT`]); when it would, -//! they allocate keys/values/hashes arrays at twice the capacity, rehash every -//! live entry into the wider mask using its cached hash, and probe there +//! it allocates keys/values/hashes arrays at twice the capacity, rehashes every +//! live entry into the wider mask using its cached hash, and probes there //! instead. `Map.set` //! is therefore total up to memory exhaustion, the same promise the //! VM and the Rust backend make. Capacity is per-map state in the @@ -37,7 +44,7 @@ //! What the growth buys is an invariant: **a map never fills**. After //! any insert, occupancy is at most three quarters of capacity, so //! some slot is always free and every probe loop terminates on it. -//! The two insert helpers still carry their wrap guard, +//! The insert helper still carries its wrap guard, //! and it is now a backstop rather than a limit — reaching it means //! the growth above it did not happen, which is a compiler bug, and //! trapping is how that bug surfaces instead of hanging. The rehash @@ -69,6 +76,10 @@ use super::types::{MapSlots, TypeRegistry}; use super::wat_helper; mod order; +mod versions; + +use versions::{VersionedWrite, emit_versioned_write}; +pub(super) use versions::{emit_no_diff, emit_reroot_local}; /// Initial bucket count — power of two so masking with `cap-1` /// instead of `i32.rem_u` works, and every doubling keeps it one. @@ -78,9 +89,7 @@ mod order; /// fresh header map per `Http.get` call and per inbound request. /// Sixteen buckets cost two 16-element arrays; the old fixed 16384 /// cost two 16384-element ones, roughly 128 KB of zeroes per empty -/// map. It also bounds the clone-on-write `set`, which copies `cap` -/// slots on every call — a three-key map used to pay 16384 element -/// copies per insert. Doubling makes the total cost of reaching `n` +/// map. Doubling makes the total cost of reaching `n` /// entries linear in `n`, so starting small costs a few extra /// rehashes early and nothing after that. /// @@ -124,15 +133,15 @@ pub(super) struct KeyHelpers { #[derive(Debug, Clone, Copy)] pub(super) struct MapKVHelpers { pub(super) empty: u32, - /// Clone-on-write `set` — allocates fresh keys/values/hashes arrays, - /// `array.copy`s them, mutates the copies. Used when `ir::alias` - /// flags the call site's map slot as alias-prone. + /// `set(m, k, v) -> m'`. Writes the bucket in place and returns a new + /// version over the same arrays; `m` stays valid through the diff the + /// write leaves on it (see `maps/versions.rs`). A grow writes into new + /// arrays instead and leaves `m` untouched. pub(super) set: u32, - /// In-place `set` — probes the source map's keys/values/hashes arrays - /// directly, `array.set`s into them, returns a struct.new wrapping - /// the same arrays with the updated size. Sound only when the IR - /// alias pass + last-use proves the map slot is uniquely owned. - pub(super) set_in_place: u32, + /// `reroot(m) -> m`. Makes `m` the version that owns its arrays' + /// contents. Every helper that reads a map's arrays calls it first, and + /// so must any code outside this module that reads them. + pub(super) reroot: u32, pub(super) get: u32, pub(super) len: u32, /// `get_or_default(m, k, default) -> V`. Fused shape that backs @@ -157,12 +166,11 @@ pub(super) struct MapKVHelpers { pub(super) keys: u32, /// `values(m) -> List` in the same key-derived order as `keys`. pub(super) values: u32, - /// `remove(m, k) -> m`. Linear-probe locate of `k`, then a + /// `remove(m, k) -> m'`. Linear-probe locate of `k`, then a /// backwards-shift scan over the rest of the probe run so every /// entry that probed past the emptied slot is still found by a - /// later `get` — see `emit_map_remove`. Mutates `m` in place - /// and returns the same handle (Aver semantics: same shape as - /// `set`, the returned ref is structurally equal). + /// later `get` — see `emit_map_remove`. Each bucket it writes is a + /// versioned write, so `m` keeps its entries, as it does for `set`. pub(super) remove: u32, /// `entries(m) -> List>` in canonical key order. pub(super) entries: u32, @@ -206,12 +214,8 @@ pub(super) struct MapHelperRegistry { #[derive(Debug, Clone, Copy)] struct MapKVTypeIdx { empty: u32, - /// Same wasm-fn type as `set_in_place` — `(map, k, v) -> map`. - /// Two distinct type entries because the fn-type table is - /// indexed by type-idx, not shape, and `assign_slots` writes - /// each helper to its own slot. set: u32, - set_in_place: u32, + reroot: u32, get: u32, len: u32, get_or_default: u32, @@ -537,7 +541,7 @@ impl MapHelperRegistry { *next_type_idx += 1; let set_type_idx = *next_type_idx; *next_type_idx += 1; - let set_in_place_type_idx = *next_type_idx; + let reroot_type_idx = *next_type_idx; *next_type_idx += 1; let get_type_idx = *next_type_idx; *next_type_idx += 1; @@ -569,7 +573,7 @@ impl MapHelperRegistry { *next_wasm_fn_idx += 1; let set_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; - let set_in_place_fn = *next_wasm_fn_idx; + let reroot_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; let get_fn = *next_wasm_fn_idx; *next_wasm_fn_idx += 1; @@ -632,7 +636,7 @@ impl MapHelperRegistry { MapKVHelpers { empty: empty_fn, set: set_fn, - set_in_place: set_in_place_fn, + reroot: reroot_fn, get: get_fn, len: len_fn, get_or_default: god_fn, @@ -653,7 +657,7 @@ impl MapHelperRegistry { MapKVTypeIdx { empty: empty_type_idx, set: set_type_idx, - set_in_place: set_in_place_type_idx, + reroot: reroot_type_idx, get: get_type_idx, len: len_type_idx, get_or_default: god_type_idx, @@ -682,9 +686,9 @@ impl MapHelperRegistry { self.kv.get(canonical).copied() } - /// `(wasm fn idx, name)` for the two insert helpers of every - /// registered `Map`, ascending by index — the shape the - /// `name` section's function subsection wants. + /// `(wasm fn idx, name)` for the insert helper of every registered + /// `Map`, ascending by index — the shape the `name` section's + /// function subsection wants. /// /// These are the only helpers that can trap, and they can only do /// it on a broken invariant: the table grows before it fills, so a @@ -693,7 +697,7 @@ impl MapHelperRegistry { /// which map's insert stopped and that a stop there is a bug, and /// the engine's backtrace reads it back. pub(super) fn capacity_helper_names(&self) -> Vec<(u32, String)> { - let mut named = Vec::with_capacity(self.kv_order.len() * 2); + let mut named = Vec::with_capacity(self.kv_order.len()); for canonical in &self.kv_order { let Some(h) = self.kv.get(canonical) else { continue; @@ -702,12 +706,6 @@ impl MapHelperRegistry { h.set, format!("{CAPACITY_HELPER_NAME_PREFIX}{canonical} (table grows; a stop here is a resize bug)"), )); - named.push(( - h.set_in_place, - format!( - "{CAPACITY_HELPER_NAME_PREFIX}{canonical} in place (table grows; a stop here is a resize bug)" - ), - )); } named.sort_by_key(|(idx, _)| *idx); named @@ -777,10 +775,10 @@ impl MapHelperRegistry { // empty : () -> Map types.ty().function([], [map_ref]); - // set : (Map, K, V) -> Map (clone-on-write) - types.ty().function([map_ref, k_val, v_val], [map_ref]); - // set_in_place : (Map, K, V) -> Map (alias-free fast path) + // set : (Map, K, V) -> Map types.ty().function([map_ref, k_val, v_val], [map_ref]); + // reroot : (Map) -> Map + types.ty().function([map_ref], [map_ref]); // get : (Map, K) -> Option types.ty().function([map_ref, k_val], [opt_ref]); // len : (Map) -> i64 @@ -874,7 +872,7 @@ impl MapHelperRegistry { let t = self.kv_type_indices[canonical]; funcs.function(t.empty); funcs.function(t.set); - funcs.function(t.set_in_place); + funcs.function(t.reroot); funcs.function(t.get); funcs.function(t.len); funcs.function(t.get_or_default); @@ -977,28 +975,47 @@ impl MapHelperRegistry { )) })?; let helpers = self.kv[canonical]; + let reroot = helpers.reroot; codes.function(&emit_map_empty(canonical, registry)?); - codes.function(&emit_map_set(canonical, registry, key_h)?); - codes.function(&emit_map_set_in_place(canonical, registry, key_h)?); - codes.function(&emit_map_get(canonical, registry, key_h)?); + codes.function(&emit_map_set(canonical, registry, key_h, reroot)?); + codes.function(&versions::emit_map_reroot( + canonical, + registry, + slots_for(canonical, registry)?, + )?); + codes.function(&emit_map_get(canonical, registry, key_h, reroot)?); codes.function(&emit_map_len(canonical, registry)?); - codes.function(&emit_map_get_or_default(canonical, registry, key_h)?); - codes.function(&emit_map_get_pair(canonical, registry, key_h)?); + codes.function(&emit_map_get_or_default( + canonical, registry, key_h, reroot, + )?); + codes.function(&emit_map_get_pair(canonical, registry, key_h, reroot)?); codes.function(&emit_map_order_sift(canonical, registry, cmp_fn)?); codes.function(&emit_map_order_slots( canonical, registry, helpers.order_sift, + reroot, + )?); + codes.function(&emit_map_keys( + canonical, + registry, + helpers.order_slots, + reroot, )?); - codes.function(&emit_map_keys(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_values(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_remove(canonical, registry, key_h)?); - codes.function(&emit_map_entries(canonical, registry, helpers.order_slots)?); - codes.function(&emit_map_from_list( + codes.function(&emit_map_values( canonical, registry, - helpers.set_in_place, + helpers.order_slots, + reroot, )?); + codes.function(&emit_map_remove(canonical, registry, key_h, reroot)?); + codes.function(&emit_map_entries( + canonical, + registry, + helpers.order_slots, + reroot, + )?); + codes.function(&emit_map_from_list(canonical, registry, helpers.set)?); // Structural eq + commutative hash for `Map`. V's // hash + eq fn idxs come from `all_key_helpers` (same // table that drives K dispatch — V is just another @@ -1011,8 +1028,11 @@ impl MapHelperRegistry { key_h, v_helpers, helpers.get, + reroot, + )?); + codes.function(&emit_map_hash( + canonical, registry, key_h, v_helpers, reroot, )?); - codes.function(&emit_map_hash(canonical, registry, key_h, v_helpers)?); } Ok(()) } @@ -1499,65 +1519,27 @@ fn emit_map_empty(canonical: &str, registry: &TypeRegistry) -> Result map`. Linear-probing open-addressing insert over -/// a clone of the map's arrays. Returns a fresh map struct wrapping -/// them; the input map is never observed mutated. +/// `set(map, k, v) -> map'`. Linear-probing open-addressing insert. +/// +/// Settles which arrays to probe (growing first if this entry would +/// overfill the table), then linear-probes from the key's home bucket — +/// an empty slot inserts, a matching key updates. Into the map's own +/// arrays the write is a versioned one (`maps/versions.rs`): the bucket's +/// old contents go on `map`'s diff and the result is a new version over the +/// same arrays, so `map` is unchanged as a value and nothing is copied. A +/// grow has already moved every entry into new arrays, which only the +/// result sees, so it writes into them directly. fn emit_map_set( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, -) -> Result { - emit_map_insert(canonical, registry, keyh, TableSource::Clone) -} - -/// `set_in_place(map, k, v) -> map`. Same insert as [`emit_map_set`] -/// without the entry-time copy of `keys` / `values` / `hashes` — the caller has -/// proven the map's arrays are uniquely owned. The returned struct -/// still re-wraps the arrays with the updated size; callers expect a -/// fresh map handle either way, which is also what lets a grow swap -/// the arrays out from under the old one. -fn emit_map_set_in_place( - canonical: &str, - registry: &TypeRegistry, - keyh: KeyHelpers, -) -> Result { - emit_map_insert(canonical, registry, keyh, TableSource::Owned) -} - -/// The body both insert helpers share: settle which arrays to probe -/// (growing first if this entry would overfill the table), then -/// linear-probe from the key's home bucket — empty slot inserts, -/// matching key updates. The two differ only in [`TableSource`], which -/// the prologue reads; every instruction after it is the same, so it -/// is written once. -fn emit_map_insert( - canonical: &str, - registry: &TypeRegistry, - keyh: KeyHelpers, - source: TableSource, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -1577,17 +1559,25 @@ fn emit_map_insert( nullable: true, heap_type: HeapType::Concrete(slots.hashes_array), }); - // params: 0=map, 1=k, 2=v + let map_ref = ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }); + let diff_ref = ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.diff), + }); + // params: 0=map (then the new version, after a versioned write), + // 1=k, 2=v // locals: 3=cap, 4=mask, 5=idx, 6=keys, 7=values, 8=cur_key, // 9=home, 10=src_keys, 11=src_values, 12=i, - // 13=hashes, 14=src_hashes, 15=query_hash + // 13=hashes, 14=src_hashes, 15=query_hash, 16=diff, 17=next // - // 6/7 are the arrays the probe writes to — a clone, the map's own, - // or a wider set a grow just filled. 10/11 are the map's arrays as - // handed in, which the grow rehashes out of and the clone copies - // from. 13/14 are their parallel cached-hash arrays. 12 walks them; - // 15 carries a stored hash during growth, then the query hash. - // 5/8/9 are reused by both probe loops. + // 6/7/13 are the arrays the probe writes to — the map's own, or a + // wider set a grow just filled. 10/11/14 are the map's arrays as + // handed in, which the grow rehashes out of. 12 walks them; 15 + // carries a stored hash during growth, then the query hash. 5/8/9 are + // reused by both probe loops. let mut f = Function::new([ (1, ValType::I32), // 3: cap (1, ValType::I32), // 4: mask @@ -1598,13 +1588,16 @@ fn emit_map_insert( (1, ValType::I32), // 9: home (probe start bucket) (1, keys_ref), // 10: src_keys (the map's own) (1, values_ref), // 11: src_values (the map's own) - (1, ValType::I32), // 12: i (rehash / copy cursor) + (1, ValType::I32), // 12: i (rehash cursor) (1, hashes_ref), // 13: hashes (probe target) (1, hashes_ref), // 14: src_hashes (the map's own) (1, ValType::I32), // 15: stored/query hash + (1, diff_ref), // 16: diff + (1, map_ref), // 17: next version ]); - emit_insert_prologue(&mut f, slots, source); + emit_reroot_local(&mut f, reroot_fn, 0); + emit_insert_prologue(&mut f, slots); // query_hash = hash(k); idx = query_hash & mask; home = idx f.instruction(&Instruction::LocalGet(1)); @@ -1632,36 +1625,25 @@ fn emit_map_insert( f.instruction(&Instruction::LocalGet(8)); f.instruction(&Instruction::RefIsNull); f.instruction(&Instruction::If(BlockType::Empty)); - // keys[idx] = box(k) (primitive K) or k (ref K) - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(1)); - emit_box_key(&mut f, k_aver, registry); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - // values[idx] = v - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(2)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - // hashes[idx] = query_hash - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(15)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); - // return struct.new $map (map.size + 1, cap, keys, values, hashes) - f.instruction(&Instruction::LocalGet(0)); - f.instruction(&Instruction::StructGet { - struct_type_index: slots.map, - field_index: 0, - }); - f.instruction(&Instruction::I32Const(1)); - f.instruction(&Instruction::I32Add); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::StructNew(slots.map)); - f.instruction(&Instruction::Return); + let insert = |f: &mut Function| { + // keys[idx] = box(k) (primitive K) or k (ref K) + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(1)); + emit_box_key(f, k_aver, registry); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + // values[idx] = v + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + // hashes[idx] = query_hash + f.instruction(&Instruction::LocalGet(13)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(15)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }; + emit_insert_write(&mut f, slots, 1, insert); f.instruction(&Instruction::End); // else if cached_hash == query_hash && eq(unbox(cur_key), k): update @@ -1678,22 +1660,13 @@ fn emit_map_insert( f.instruction(&Instruction::LocalGet(1)); f.instruction(&Instruction::Call(keyh.eq)); f.instruction(&Instruction::If(BlockType::Empty)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(2)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - // return struct.new $map (map.size, cap, keys, values, hashes) - f.instruction(&Instruction::LocalGet(0)); - f.instruction(&Instruction::StructGet { - struct_type_index: slots.map, - field_index: 0, - }); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::StructNew(slots.map)); - f.instruction(&Instruction::Return); + let update = |f: &mut Function| { + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + }; + emit_insert_write(&mut f, slots, 0, update); f.instruction(&Instruction::End); f.instruction(&Instruction::End); @@ -1713,15 +1686,81 @@ fn emit_map_insert( Ok(f) } +/// Write bucket `idx` (local 5) of an insert with `write`, then return the +/// map it makes, `size_delta` entries larger than `map`. +/// +/// When the probe wrote into `map`'s own arrays (6 is 10), the write is a +/// versioned one and the result is the new version. After a grow the +/// arrays are new and only the result holds them, so it writes directly. +fn emit_insert_write( + f: &mut Function, + slots: MapSlots, + size_delta: i32, + write: impl Fn(&mut Function), +) { + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(10)); + f.instruction(&Instruction::RefEq); + f.instruction(&Instruction::If(BlockType::Empty)); + emit_versioned_write( + f, + slots, + &VersionedWrite { + version: 0, + diff: 16, + next: 17, + idx: 5, + cap: 3, + keys: 6, + values: 7, + hashes: 13, + }, + &write, + ); + if size_delta != 0 { + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + f.instruction(&Instruction::I32Const(size_delta)); + f.instruction(&Instruction::I32Add); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: 0, + }); + } + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::Return); + f.instruction(&Instruction::End); + write(f); + // return struct.new $map (map.size + delta, cap, keys, values, hashes) + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + if size_delta != 0 { + f.instruction(&Instruction::I32Const(size_delta)); + f.instruction(&Instruction::I32Add); + } + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(13)); + emit_no_diff(f, slots); + f.instruction(&Instruction::StructNew(slots.map)); + f.instruction(&Instruction::Return); +} + /// Settle `cap` (3), `mask` (4) and the arrays the probe will write to /// (6, 7, 13) for one insert, growing the table first when the entry about /// to be added would push occupancy past three quarters of capacity. /// -/// Growing and copying are the same act, so they are one branch each -/// and never both: a clone-on-write insert that grows allocates the -/// wider arrays and rehashes straight into them, which copies every -/// live entry exactly once. The `array.copy` in the other branch is -/// the cheaper move for the far more common insert that does not grow. +/// A grow allocates the wider arrays and rehashes straight into them, +/// which copies every live entry exactly once. Without a grow the probe +/// writes into the map's own arrays (see [`emit_insert_write`]). /// /// Rehashing rather than copying is what the wider mask requires: a /// key's bucket is `hash & (cap - 1)`, so doubling `cap` exposes one @@ -1732,7 +1771,7 @@ fn emit_map_insert( /// The map's own arrays are read out first (10, 11, 14) because both /// branches need them, and because reading them before the branch /// keeps the grow path from re-reading struct fields per entry. -fn emit_insert_prologue(f: &mut Function, slots: MapSlots, source: TableSource) { +fn emit_insert_prologue(f: &mut Function, slots: MapSlots) { // cap = map.cap; src_keys/src_values/src_hashes = map arrays f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { @@ -1891,57 +1930,12 @@ fn emit_insert_prologue(f: &mut Function, slots: MapSlots, source: TableSource) f.instruction(&Instruction::I32Const(1)); f.instruction(&Instruction::I32Sub); f.instruction(&Instruction::LocalSet(4)); - match source { - TableSource::Clone => { - // keys = array.new_default cap; array.copy keys 0 src_keys 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.keys_array)); - f.instruction(&Instruction::LocalSet(6)); - f.instruction(&Instruction::LocalGet(6)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(10)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.keys_array, - array_type_index_src: slots.keys_array, - }); - // values = array.new_default cap; array.copy values 0 src_values 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.values_array)); - f.instruction(&Instruction::LocalSet(7)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(11)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.values_array, - array_type_index_src: slots.values_array, - }); - // hashes = array.new_default cap; array.copy hashes 0 src_hashes 0 cap - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayNewDefault(slots.hashes_array)); - f.instruction(&Instruction::LocalSet(13)); - f.instruction(&Instruction::LocalGet(13)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::LocalGet(3)); - f.instruction(&Instruction::ArrayCopy { - array_type_index_dst: slots.hashes_array, - array_type_index_src: slots.hashes_array, - }); - } - TableSource::Owned => { - f.instruction(&Instruction::LocalGet(10)); - f.instruction(&Instruction::LocalSet(6)); - f.instruction(&Instruction::LocalGet(11)); - f.instruction(&Instruction::LocalSet(7)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalSet(13)); - } - } + f.instruction(&Instruction::LocalGet(10)); + f.instruction(&Instruction::LocalSet(6)); + f.instruction(&Instruction::LocalGet(11)); + f.instruction(&Instruction::LocalSet(7)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalSet(13)); f.instruction(&Instruction::End); // grow if/else } @@ -2009,6 +2003,7 @@ fn emit_map_get( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2052,6 +2047,7 @@ fn emit_map_get( ]); let _ = k_val; // cap, mask, keys, values + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2156,6 +2152,7 @@ fn emit_map_get_or_default( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2196,6 +2193,7 @@ fn emit_map_get_or_default( ]); // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2298,6 +2296,7 @@ fn emit_map_get_pair( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2337,6 +2336,7 @@ fn emit_map_get_pair( (1, ValType::I32), // 10: query_hash ]); + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2819,6 +2819,7 @@ fn emit_map_order_slots( canonical: &str, registry: &TypeRegistry, sift_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let order_idx = registry @@ -2837,6 +2838,7 @@ fn emit_map_order_slots( // params: 0=map. locals: 1=keys, 2=indices, 3=count, 4=cap, // 5=slot, 6=used, 7=start, 8=end, 9=tmp. let mut f = Function::new([(1, keys_ref), (1, order_ref), (7, ValType::I32)]); + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2966,6 +2968,7 @@ fn emit_map_keys( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, _) = super::types::parse_map_kv(canonical).unwrap(); @@ -2975,7 +2978,7 @@ fn emit_map_keys( .ok_or(WasmGcError::Validation(format!( "Map.keys: `{list_canonical}` not registered" )))?; - emit_map_walk_keys_to_list(slots, list_idx, k_aver, registry, order_slots_fn) + emit_map_walk_keys_to_list(slots, list_idx, k_aver, registry, order_slots_fn, reroot_fn) } /// `values(m) -> List`. Same shape as `keys` but pulls from @@ -2984,6 +2987,7 @@ fn emit_map_values( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (_, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -2993,7 +2997,7 @@ fn emit_map_values( .ok_or(WasmGcError::Validation(format!( "Map.values: `{list_canonical}` not registered" )))?; - emit_map_walk_values_to_list(slots, registry, list_idx, order_slots_fn) + emit_map_walk_values_to_list(slots, registry, list_idx, order_slots_fn, reroot_fn) } /// Real impl for `Map.keys` walking the keys array. Per primitive @@ -3005,6 +3009,7 @@ fn emit_map_walk_keys_to_list( k_aver: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let order_idx = registry .map_order_indices_type_idx @@ -3031,6 +3036,7 @@ fn emit_map_walk_keys_to_list( (1, list_ref), ]); // keys = map.keys + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3090,6 +3096,7 @@ fn emit_map_walk_values_to_list( registry: &TypeRegistry, list_idx: u32, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let order_idx = registry .map_order_indices_type_idx @@ -3115,6 +3122,7 @@ fn emit_map_walk_values_to_list( (1, ValType::I32), (1, list_ref), ]); + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3170,6 +3178,7 @@ fn emit_map_eq( keyh: KeyHelpers, v_helpers: Option, get_fn_idx: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3200,7 +3209,8 @@ fn emit_map_eq( heap_type: HeapType::Concrete(opt_idx), }); // Locals: 2=typed map_a, 3=typed map_b, 4=cap, 5=i, 6=keys_a, - // 7=values_a, 8=cur_key (boxed), 9=opt result, 10=v_a, 11=v_b + // 7=values_a, 8=cur_key (boxed), 9=opt result, 10=v_a, 11=v_b, + // 12=copy of keys_a, 13=copy of values_a let mut f = Function::new(vec![ (1, map_ref), (1, map_ref), @@ -3212,6 +3222,8 @@ fn emit_map_eq( (1, opt_ref), (1, v_val), (1, v_val), + (1, keys_ref), + (1, values_ref), ]); let map_heap = HeapType::Concrete(slots.map); f.instruction(&Instruction::LocalGet(0)); @@ -3237,6 +3249,7 @@ fn emit_map_eq( f.instruction(&Instruction::Return); f.instruction(&Instruction::End); // cap = a.cap; keys_a = a.keys; values_a = a.values; i = 0 + emit_reroot_local(&mut f, reroot_fn, 2); f.instruction(&Instruction::LocalGet(2)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3255,6 +3268,33 @@ fn emit_map_eq( field_index: 3, }); f.instruction(&Instruction::LocalSet(7)); + // Two versions of one lineage share their arrays, and every `get` on + // `b` below makes `b` the current one. Walk a copy of `a`'s buckets + // then, so `a` reads as itself throughout. + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 2, + }); + f.instruction(&Instruction::RefEq); + f.instruction(&Instruction::If(BlockType::Empty)); + for (local, copy, array_type) in [(6, 12, slots.keys_array), (7, 13, slots.values_array)] { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::ArrayNewDefault(array_type)); + f.instruction(&Instruction::LocalTee(copy)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::LocalGet(local)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::ArrayCopy { + array_type_index_dst: array_type, + array_type_index_src: array_type, + }); + f.instruction(&Instruction::LocalGet(copy)); + f.instruction(&Instruction::LocalSet(local)); + } + f.instruction(&Instruction::End); f.instruction(&Instruction::I32Const(0)); f.instruction(&Instruction::LocalSet(5)); // for i in 0..cap @@ -3403,6 +3443,7 @@ fn emit_map_hash( registry: &TypeRegistry, keyh: KeyHelpers, v_helpers: Option, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3434,6 +3475,7 @@ fn emit_map_hash( f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::RefCastNonNull(map_heap)); f.instruction(&Instruction::LocalSet(1)); + emit_reroot_local(&mut f, reroot_fn, 1); f.instruction(&Instruction::I32Const(0)); f.instruction(&Instruction::LocalSet(7)); f.instruction(&Instruction::LocalGet(1)); @@ -3506,16 +3548,18 @@ fn emit_map_hash( Ok(f) } -/// `remove(map, k) -> map`. Linear-probe locate the entry; if not +/// `remove(map, k) -> map'`. Linear-probe locate the entry; if not /// found, return the map unchanged. If found, empty its slot and walk /// the rest of the probe run, pulling back every entry that the new /// hole would otherwise hide from its own lookup — the backwards-shift -/// deletion for linear probing. Decrements `map.size`. Same-handle -/// return (mutates in place). +/// deletion for linear probing. Every bucket it writes is a versioned +/// write (`maps/versions.rs`), so `map` keeps its entries and the last +/// version, one entry smaller, is the result. fn emit_map_remove( canonical: &str, registry: &TypeRegistry, keyh: KeyHelpers, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3525,7 +3569,7 @@ fn emit_map_remove( // params: 0=map, 1=k. // locals: 2=cap, 3=mask, 4=keys, 5=values, 6=h, 7=i, 8=j, // 9=cur_key, 10=natural, 11=gap, 12=disp, 13=hole, - // 14=hashes, 15=query_hash. + // 14=hashes, 15=query_hash, 16=version, 17=diff, 18=next. // `7=i` is the probe index while the entry is being located, then // the hole the shift is filling — which travels as entries move. // `13=hole` keeps the slot the removed key vacated, unchanged, as @@ -3563,9 +3607,41 @@ fn emit_map_remove( }), ), // 14: hashes (1, ValType::I32), // 15: query_hash + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }), + ), // 16: version + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.diff), + }), + ), // 17: diff + ( + 1, + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(slots.map), + }), + ), // 18: next version ]); + let versioned = VersionedWrite { + version: 16, + diff: 17, + next: 18, + idx: 7, + cap: 2, + keys: 4, + values: 5, + hashes: 14, + }; // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3675,7 +3751,11 @@ fn emit_map_remove( // there. Those are exactly the entries a stop would strand. // // `hole` (local 13) keeps the slot the removal emptied — the scan's - // wrap guard, see below. The live hole travels in `i`. + // wrap guard, see below. The live hole travels in `i`. `version` + // (local 16) starts at `map` and moves to each new version a write + // makes. + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalSet(16)); f.instruction(&Instruction::LocalGet(7)); f.instruction(&Instruction::LocalSet(13)); f.instruction(&Instruction::LocalGet(7)); @@ -3732,22 +3812,24 @@ fn emit_map_remove( f.instruction(&Instruction::I32GeU); f.instruction(&Instruction::If(BlockType::Empty)); // shift: keys[i] = next; values[i] = values[j]; hashes[i] = hashes[j] - f.instruction(&Instruction::LocalGet(4)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(9)); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(5)); - f.instruction(&Instruction::LocalGet(8)); - f.instruction(&Instruction::ArrayGet(slots.values_array)); - f.instruction(&Instruction::ArraySet(slots.values_array)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(8)); - f.instruction(&Instruction::ArrayGet(slots.hashes_array)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); + emit_versioned_write(&mut f, slots, &versioned, |f| { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(9)); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(8)); + f.instruction(&Instruction::ArrayGet(slots.values_array)); + f.instruction(&Instruction::ArraySet(slots.values_array)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(8)); + f.instruction(&Instruction::ArrayGet(slots.hashes_array)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }); // i = j — the slot just vacated is the new hole. Its stale key stays // in the array until the next move overwrites it or the `keys[i] = // null` below clears it, and no lookup can reach it in between. @@ -3770,17 +3852,20 @@ fn emit_map_remove( // box / String / record / carrier / List / Vector concrete idx, // nominal root for sum K). let null_heap = key_storage_null_heap(k_aver, registry); - f.instruction(&Instruction::LocalGet(4)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::RefNull(null_heap)); - f.instruction(&Instruction::ArraySet(slots.keys_array)); - f.instruction(&Instruction::LocalGet(14)); - f.instruction(&Instruction::LocalGet(7)); - f.instruction(&Instruction::I32Const(0)); - f.instruction(&Instruction::ArraySet(slots.hashes_array)); + emit_versioned_write(&mut f, slots, &versioned, |f| { + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::RefNull(null_heap)); + f.instruction(&Instruction::ArraySet(slots.keys_array)); + f.instruction(&Instruction::LocalGet(14)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::I32Const(0)); + f.instruction(&Instruction::ArraySet(slots.hashes_array)); + }); - // map.size = map.size - 1 - f.instruction(&Instruction::LocalGet(0)); + // The last version is new and nobody else holds it yet: + // version.size = map.size - 1 + f.instruction(&Instruction::LocalGet(16)); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3793,8 +3878,8 @@ fn emit_map_remove( field_index: 0, }); - // return map - f.instruction(&Instruction::LocalGet(0)); + // return the last version + f.instruction(&Instruction::LocalGet(16)); f.instruction(&Instruction::End); Ok(f) } @@ -3804,6 +3889,7 @@ fn emit_map_entries( canonical: &str, registry: &TypeRegistry, order_slots_fn: u32, + reroot_fn: u32, ) -> Result { let slots = slots_for(canonical, registry)?; let (k_aver, v_aver) = super::types::parse_map_kv(canonical).unwrap(); @@ -3849,6 +3935,7 @@ fn emit_map_entries( (1, ValType::I32), (1, lt_ref), ]); + emit_reroot_local(&mut f, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3910,17 +3997,10 @@ fn emit_map_entries( /// `from_list(l) -> Map`. Walks `l` from head to tail, /// struct.get's the (K, V) from each tuple, calls the per-(K, V) -/// `set_in_place` helper to insert. Allocates a fresh empty map (via -/// the per-(K, V) `empty` shape inlined: cap = INITIAL_CAP, fresh keys -/// and values arrays) and returns it. -/// -/// In place rather than clone-on-write because the map is allocated -/// here, is held only by this frame's local, and is not handed to -/// anyone until the walk finishes — the uniqueness the alias pass -/// proves at a `Map.set` call site is a fact of this body's shape. -/// Copying instead would make `Map.fromList` quadratic: one full array -/// copy per pair. With growth in the insert helper that would be the -/// only remaining size cliff a map has. +/// `set` helper to insert. Allocates a fresh empty map (via the +/// per-(K, V) `empty` shape inlined: cap = INITIAL_CAP, fresh keys +/// and values arrays) and returns it. `set` writes in place, so the +/// walk is linear in the list. fn emit_map_from_list( canonical: &str, registry: &TypeRegistry, @@ -3969,6 +4049,7 @@ fn emit_map_from_list( f.instruction(&Instruction::ArrayNewDefault(slots.values_array)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(slots.hashes_array)); + emit_no_diff(&mut f, slots); f.instruction(&Instruction::StructNew(slots.map)); f.instruction(&Instruction::LocalSet(2)); // cur = l diff --git a/src/codegen/wasm_gc/maps/versions.rs b/src/codegen/wasm_gc/maps/versions.rs new file mode 100644 index 000000000..2948ae8c3 --- /dev/null +++ b/src/codegen/wasm_gc/maps/versions.rs @@ -0,0 +1,324 @@ +//! Versions of one map that share its arrays. +//! +//! A `Map` value is a `$map` struct over three arrays. `Map.set` and +//! `Map.remove` write into those arrays in place and return a new `$map` +//! struct over the same arrays, so an update costs a probe, not a copy of +//! the table. The map it was given stays a valid value: before the write, +//! the helper records what the bucket held in a `$diff` struct and hangs it +//! on the old `$map` (`diff` field). The old version is then "the new +//! version, except that bucket `idx` holds `key`, `value`, `hash`". +//! +//! Exactly one version of a lineage owns the arrays' current contents: the +//! one whose `diff` is null. [`emit_map_reroot`] makes a given version that +//! one. It follows the `diff` chain to the current version and walks back, +//! swapping each recorded bucket into the arrays and hanging the swapped-out +//! contents on the version it just left, so every version stays readable +//! and the chain now points the other way. Every helper that reads or writes +//! a map's arrays reroots first; `len` reads only the version's own `size`. +//! +//! A program that uses each map once, as the generated loop and most +//! recursive builders do, never has more than one diff to undo, so an update +//! is O(1). A program that keeps an old version and reads it again pays for +//! the buckets written since, once, and the version it moved away from pays +//! the same to come back. A grow allocates new arrays and leaves the old +//! ones to the old version, so no diff crosses a grow. +//! +//! This is the persistent-array technique of Baker ("Shallow binding makes +//! functional arrays fast", 1991), as in Conchon and Filliâtre's +//! "A Persistent Union-Find Data Structure" (2007). +//! +//! Two versions of one lineage cannot both be current. A helper that reads +//! two maps at once (`eq`) therefore copies one side when both share arrays. +//! The host reads a map's buckets directly only for the sockets of +//! `Tcp.poll` and the wait set of `Wait.poll`; those arguments are rerooted +//! before the import is called. + +use wasm_encoder::{BlockType, Function, HeapType, Instruction, RefType, ValType}; + +use super::super::WasmGcError; +use super::super::types::{MapSlots, TypeRegistry}; +use super::key_storage_val_type; + +/// `$map` field holding the version's diff; null on the current version. +const DIFF_FIELD: u32 = 5; + +const DIFF_NEXT: u32 = 0; +const DIFF_IDX: u32 = 1; +const DIFF_KEY: u32 = 2; +const DIFF_VALUE: u32 = 3; +const DIFF_HASH: u32 = 4; + +fn nullable(idx: u32) -> ValType { + ValType::Ref(RefType { + nullable: true, + heap_type: HeapType::Concrete(idx), + }) +} + +/// Push a null diff: the operand every `struct.new $map` of a current +/// version ends with. +pub(in crate::codegen::wasm_gc) fn emit_no_diff(f: &mut Function, slots: MapSlots) { + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.diff))); +} + +/// `local.get map; call reroot; local.set map`: make the version in +/// `map_local` the current one before its arrays are read. +pub(in crate::codegen::wasm_gc) fn emit_reroot_local( + f: &mut Function, + reroot_fn: u32, + map_local: u32, +) { + f.instruction(&Instruction::LocalGet(map_local)); + f.instruction(&Instruction::Call(reroot_fn)); + f.instruction(&Instruction::LocalSet(map_local)); +} + +/// `reroot(map) -> map`: make `map` the version that owns its arrays' +/// contents, and return it. +pub(super) fn emit_map_reroot( + canonical: &str, + registry: &TypeRegistry, + slots: MapSlots, +) -> Result { + let (k_aver, v_aver) = super::super::types::parse_map_kv(canonical).ok_or_else(|| { + WasmGcError::Validation(format!("Map.reroot: bad canonical `{canonical}`")) + })?; + let v_val = super::super::types::aver_to_wasm(v_aver, Some(registry))?.ok_or_else(|| { + WasmGcError::Validation(format!( + "Map value type `{v_aver}` has no wasm representation" + )) + })?; + let map_ref = nullable(slots.map); + let diff_ref = nullable(slots.diff); + // params: 0=map. locals: 1=prev, 2=cur, 3=diff, 4=next, 5=current + // version, 6=version being restored, 7=idx, 8=keys, 9=values, + // 10=hashes, 11/12/13 = the bucket's key/value/hash being swapped out. + let mut f = Function::new([ + (1, map_ref), // 1: prev + (1, map_ref), // 2: cur + (1, diff_ref), // 3: diff + (1, map_ref), // 4: next + (1, map_ref), // 5: current version + (1, map_ref), // 6: version being restored + (1, ValType::I32), // 7: idx + (1, nullable(slots.keys_array)), // 8: keys + (1, nullable(slots.values_array)), // 9: values + (1, nullable(slots.hashes_array)), // 10: hashes + (1, key_storage_val_type(k_aver, registry)?), // 11: key out + (1, v_val), // 12: value out + (1, ValType::I32), // 13: hash out + ]); + let get_map = |f: &mut Function, field| { + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: field, + }); + }; + let get_diff = |f: &mut Function, field| { + f.instruction(&Instruction::StructGet { + struct_type_index: slots.diff, + field_index: field, + }); + }; + let set_diff = |f: &mut Function, field| { + f.instruction(&Instruction::StructSet { + struct_type_index: slots.diff, + field_index: field, + }); + }; + + // Already current: nothing to do. This is every call on a map used once. + f.instruction(&Instruction::LocalGet(0)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::If(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::Return); + f.instruction(&Instruction::End); + + // Walk to the current version, turning each diff's `next` round to + // point at the version before it. `prev` ends on the last stale + // version, whose diff is relative to the current one. + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.map))); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::LocalSet(2)); + f.instruction(&Instruction::Block(BlockType::Empty)); + f.instruction(&Instruction::Loop(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(2)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::LocalTee(3)); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::BrIf(1)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalSet(4)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(1)); + set_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(4)); + f.instruction(&Instruction::LocalSet(2)); + f.instruction(&Instruction::Br(0)); + f.instruction(&Instruction::End); + f.instruction(&Instruction::End); + + // Every version in the chain shares the current version's arrays. + f.instruction(&Instruction::LocalGet(2)); + f.instruction(&Instruction::LocalSet(5)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 2); + f.instruction(&Instruction::LocalSet(8)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 3); + f.instruction(&Instruction::LocalSet(9)); + f.instruction(&Instruction::LocalGet(5)); + get_map(&mut f, 4); + f.instruction(&Instruction::LocalSet(10)); + + // Walk back towards `map`, one version at a time: put that version's + // bucket back into the arrays, and keep what was there as the diff of + // the version just left, pointing forward to the restored one. + f.instruction(&Instruction::Block(BlockType::Empty)); + f.instruction(&Instruction::Loop(BlockType::Empty)); + f.instruction(&Instruction::LocalGet(1)); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::BrIf(1)); + f.instruction(&Instruction::LocalGet(1)); + f.instruction(&Instruction::LocalSet(6)); + f.instruction(&Instruction::LocalGet(6)); + get_map(&mut f, DIFF_FIELD); + f.instruction(&Instruction::LocalSet(3)); + // prev = the version before this one (the reversed link). + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalSet(1)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, DIFF_IDX); + f.instruction(&Instruction::LocalSet(7)); + // Swap the bucket out. + for (array_local, array_type, out_local) in [ + (8, slots.keys_array, 11), + (9, slots.values_array, 12), + (10, slots.hashes_array, 13), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::ArrayGet(array_type)); + f.instruction(&Instruction::LocalSet(out_local)); + } + // Put the restored version's bucket in. + for (array_local, array_type, field) in [ + (8, slots.keys_array, DIFF_KEY), + (9, slots.values_array, DIFF_VALUE), + (10, slots.hashes_array, DIFF_HASH), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(7)); + f.instruction(&Instruction::LocalGet(3)); + get_diff(&mut f, field); + f.instruction(&Instruction::ArraySet(array_type)); + } + // The diff now describes the version just left, relative to the + // restored one. + for (out_local, field) in [(11, DIFF_KEY), (12, DIFF_VALUE), (13, DIFF_HASH)] { + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(out_local)); + set_diff(&mut f, field); + } + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::LocalGet(6)); + set_diff(&mut f, DIFF_NEXT); + f.instruction(&Instruction::LocalGet(5)); + f.instruction(&Instruction::LocalGet(3)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(6)); + emit_no_diff(&mut f, slots); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(6)); + f.instruction(&Instruction::LocalSet(5)); + f.instruction(&Instruction::Br(0)); + f.instruction(&Instruction::End); + f.instruction(&Instruction::End); + + f.instruction(&Instruction::LocalGet(0)); + f.instruction(&Instruction::End); + Ok(f) +} + +/// Locals a versioned write reads and writes. `version` holds the current +/// version before the write and the new one after it. +pub(super) struct VersionedWrite { + pub version: u32, + pub diff: u32, + pub next: u32, + pub idx: u32, + pub cap: u32, + pub keys: u32, + pub values: u32, + pub hashes: u32, +} + +/// Write one bucket of the current version in `w.version` as a new version. +/// +/// Records the bucket's key, value and hash on the old version, lets +/// `write` store the new ones (it runs with nothing on the stack and must +/// leave nothing), and moves `w.version` to a new `$map` over the same +/// arrays with the old version's size. The caller adjusts the size of the +/// last version it makes. +pub(super) fn emit_versioned_write( + f: &mut Function, + slots: MapSlots, + w: &VersionedWrite, + write: impl FnOnce(&mut Function), +) { + f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.map))); + f.instruction(&Instruction::LocalGet(w.idx)); + for (array_local, array_type) in [ + (w.keys, slots.keys_array), + (w.values, slots.values_array), + (w.hashes, slots.hashes_array), + ] { + f.instruction(&Instruction::LocalGet(array_local)); + f.instruction(&Instruction::LocalGet(w.idx)); + f.instruction(&Instruction::ArrayGet(array_type)); + } + f.instruction(&Instruction::StructNew(slots.diff)); + f.instruction(&Instruction::LocalSet(w.diff)); + + write(f); + + f.instruction(&Instruction::LocalGet(w.version)); + f.instruction(&Instruction::StructGet { + struct_type_index: slots.map, + field_index: 0, + }); + f.instruction(&Instruction::LocalGet(w.cap)); + f.instruction(&Instruction::LocalGet(w.keys)); + f.instruction(&Instruction::LocalGet(w.values)); + f.instruction(&Instruction::LocalGet(w.hashes)); + emit_no_diff(f, slots); + f.instruction(&Instruction::StructNew(slots.map)); + f.instruction(&Instruction::LocalSet(w.next)); + f.instruction(&Instruction::LocalGet(w.diff)); + f.instruction(&Instruction::LocalGet(w.next)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.diff, + field_index: DIFF_NEXT, + }); + f.instruction(&Instruction::LocalGet(w.version)); + f.instruction(&Instruction::LocalGet(w.diff)); + f.instruction(&Instruction::StructSet { + struct_type_index: slots.map, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::LocalGet(w.next)); + f.instruction(&Instruction::LocalSet(w.version)); +} diff --git a/src/codegen/wasm_gc/module.rs b/src/codegen/wasm_gc/module.rs index 15d86e01a..5124496cd 100644 --- a/src/codegen/wasm_gc/module.rs +++ b/src/codegen/wasm_gc/module.rs @@ -2198,6 +2198,15 @@ pub(super) fn emit_module_with( headers_values_array_type_idx: map_slots.values_array, headers_hashes_array_type_idx: map_slots.hashes_array, headers_map_type_idx: map_slots.map, + headers_map_diff_type_idx: map_slots.diff, + headers_map_reroot_fn: map_helpers + .kv_helpers("Map>") + .ok_or_else(|| { + WasmGcError::Validation( + "Http headers need the Map> helpers".into(), + ) + })? + .reroot, list_string_type_idx: list_string_idx, option_list_string_type_idx: opt_list_string_idx, aint_from_i64_fn_idx: registry.aint_from_i64_fn_idx, @@ -3614,6 +3623,15 @@ pub(super) fn emit_module_with( headers_values_array_type_idx: map_slots.values_array, headers_hashes_array_type_idx: map_slots.hashes_array, headers_map_type_idx: map_slots.map, + headers_map_diff_type_idx: map_slots.diff, + headers_map_reroot_fn: map_helpers + .kv_helpers("Map>") + .ok_or_else(|| { + WasmGcError::Validation( + "Http headers need the Map> helpers".into(), + ) + })? + .reroot, list_string_type_idx: list_string_idx, option_list_string_type_idx: opt_list_string_idx, aint_to_i64_checked_fn_idx: registry.aint_to_i64_checked_fn_idx, @@ -6810,6 +6828,48 @@ fn emit_user_types( element_type: wasm_encoder::StorageType::Val(hashes_ref), mutable: true, }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::Ref( + wasm_encoder::RefType { + nullable: true, + heap_type: wasm_encoder::HeapType::Concrete(slots.diff), + }, + )), + mutable: true, + }, + ]), + )); + // One bucket of an older version: the version it leads to, the + // bucket, and the key, value and hash that bucket held before the + // newer version wrote over it (see `maps.rs`). + entries.push(( + slots.diff, + mk_struct(vec![ + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::Ref( + wasm_encoder::RefType { + nullable: true, + heap_type: wasm_encoder::HeapType::Concrete(slots.map), + }, + )), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::I32), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(key_storage_val), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(v_val), + mutable: true, + }, + wasm_encoder::FieldType { + element_type: wasm_encoder::StorageType::Val(ValType::I32), + mutable: true, + }, ]), )); } @@ -10652,6 +10712,7 @@ fn emit_factory_map_string_list_string_empty( f.instruction(&Instruction::RefNull(wasm_encoder::HeapType::Concrete( slots.hashes_array, ))); + super::maps::emit_no_diff(&mut f, slots); f.instruction(&Instruction::StructNew(slots.map)); f.instruction(&Instruction::End); Ok(f) @@ -10903,6 +10964,13 @@ fn emit_handler_wrapper( field_index: 2, }); f.instruction(&Instruction::LocalSet(9)); + let headers_reroot = fn_map + .map_helpers_lookup("Map>") + .ok_or(WasmGcError::Validation( + "aver_http_handle wrapper requires the Map> helpers".into(), + ))? + .reroot; + super::maps::emit_reroot_local(&mut f, headers_reroot, 9); // Read map cap + arrays into iteration slots. f.instruction(&Instruction::LocalGet(9)); diff --git a/src/codegen/wasm_gc/types.rs b/src/codegen/wasm_gc/types.rs index 44e654103..4ffdb074d 100644 --- a/src/codegen/wasm_gc/types.rs +++ b/src/codegen/wasm_gc/types.rs @@ -348,8 +348,13 @@ pub(super) struct MapSlots { /// `(array (mut i32))` — cached complete key hash per occupied bucket. pub(super) hashes_array: u32, /// `(struct (mut i32 size) (mut i32 cap) (mut keys_ref) (mut values_ref) - /// (mut hashes_ref))`. + /// (mut hashes_ref) (mut diff_ref))`. A null `diff` marks the + /// version that owns the arrays' current contents; see `maps.rs`. pub(super) map: u32, + /// `(struct (mut map_ref next) (mut i32 idx) (mut K key) (mut V value) + /// (mut i32 hash))` — what one bucket held in an older version + /// of a map whose arrays a newer version has since written to. + pub(super) diff: u32, } impl TypeRegistry { @@ -1103,7 +1108,8 @@ impl TypeRegistry { next_idx += 1; } } - // Allocate four slots: keys_array, values_array, hashes_array, map. + // Allocate five slots: keys_array, values_array, hashes_array, map, + // diff. // Order: arrays first so the struct (higher idx) can // reference them without crossing rec-group boundaries. let keys_array = next_idx; @@ -1114,6 +1120,8 @@ impl TypeRegistry { next_idx += 1; let map = next_idx; next_idx += 1; + let diff = next_idx; + next_idx += 1; map_types.insert( canonical.clone(), MapSlots { @@ -1121,6 +1129,7 @@ impl TypeRegistry { values_array, hashes_array, map, + diff, }, ); map_order.push(canonical); diff --git a/src/codegen/wasm_gc/wasip2_http.rs b/src/codegen/wasm_gc/wasip2_http.rs index c59d50ad0..effaf76ac 100644 --- a/src/codegen/wasm_gc/wasip2_http.rs +++ b/src/codegen/wasm_gc/wasip2_http.rs @@ -72,6 +72,10 @@ pub(super) struct HttpGetIndices { pub headers_values_array_type_idx: u32, pub headers_hashes_array_type_idx: u32, pub headers_map_type_idx: u32, + /// The map's `$diff` struct: a fresh map carries a null one. + pub headers_map_diff_type_idx: u32, + /// The map's `reroot` helper, called before its buckets are read. + pub headers_map_reroot_fn: u32, /// `List` cons-cell type idx. Each header value lands /// either in a singleton `[value]` list or prepended onto the /// existing list when the same field-key reappears (Set-Cookie @@ -773,6 +777,7 @@ pub(super) fn emit_http_get(indices: &HttpGetIndices, h: &HttpGetHelperFns) -> F // but the dispatcher passes an empty map, so it's a no-op // beyond the cap-iter (~16k iterations of "is keys[i] null? // yes, skip"). Acceptable for v1 PoC. + super::maps::emit_reroot_local(&mut f, indices.headers_map_reroot_fn, p_headers); f.instruction(&Instruction::LocalGet(p_headers)); f.instruction(&Instruction::StructGet { struct_type_index: indices.headers_map_type_idx, @@ -1523,6 +1528,9 @@ pub(super) fn emit_http_get(indices: &HttpGetIndices, h: &HttpGetHelperFns) -> F f.instruction(&Instruction::ArrayNewDefault(values_arr_idx)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(hashes_arr_idx)); + f.instruction(&Instruction::RefNull(HeapType::Concrete( + indices.headers_map_diff_type_idx, + ))); f.instruction(&Instruction::StructNew(map_idx)); f.instruction(&Instruction::LocalSet(l_h_map)); diff --git a/src/codegen/wasm_gc/wasip2_http_handler.rs b/src/codegen/wasm_gc/wasip2_http_handler.rs index da77ab06a..8df1b878c 100644 --- a/src/codegen/wasm_gc/wasip2_http_handler.rs +++ b/src/codegen/wasm_gc/wasip2_http_handler.rs @@ -99,6 +99,10 @@ pub(super) struct ServerHandlerIndices { pub headers_values_array_type_idx: u32, pub headers_hashes_array_type_idx: u32, pub headers_map_type_idx: u32, + /// The map's `$diff` struct: a fresh map carries a null one. + pub headers_map_diff_type_idx: u32, + /// The map's `reroot` helper, called before its buckets are read. + pub headers_map_reroot_fn: u32, /// `List` cons-cell type idx — head = String ref, tail /// = list ref. pub list_string_type_idx: u32, @@ -699,6 +703,9 @@ pub(super) fn emit_aver_http_handle( f.instruction(&Instruction::ArrayNewDefault(values_arr_idx)); f.instruction(&Instruction::I32Const(INITIAL_CAP)); f.instruction(&Instruction::ArrayNewDefault(hashes_arr_idx)); + f.instruction(&Instruction::RefNull(HeapType::Concrete( + indices.headers_map_diff_type_idx, + ))); f.instruction(&Instruction::StructNew(map_idx)); f.instruction(&Instruction::LocalSet(l_req_headers_map)); @@ -1167,6 +1174,7 @@ pub(super) fn emit_aver_http_handle( } // Walk response headers map → fields.append. + super::maps::emit_reroot_local(&mut f, indices.headers_map_reroot_fn, l_resp_headers_map); f.instruction(&Instruction::LocalGet(l_resp_headers_map)); f.instruction(&Instruction::StructGet { struct_type_index: map_idx, diff --git a/tests/wasm_gc_map_versions_spec.rs b/tests/wasm_gc_map_versions_spec.rs new file mode 100644 index 000000000..2cc6d4de1 --- /dev/null +++ b/tests/wasm_gc_map_versions_spec.rs @@ -0,0 +1,224 @@ +//! Versions of one wasm-gc map. +//! +//! `Map.set` and `Map.remove` write into the map's arrays in place and +//! leave the map they were given valid through a diff (see +//! `src/codegen/wasm_gc/maps/versions.rs`). These tests keep every older +//! version in use after newer ones are made from it: across updates, +//! inserts, removes that shift a probe run, growth, a long chain of +//! overwrites, two branches from one base, and equality between versions +//! that share arrays. Each program runs on the VM and on wasm-gc, and both +//! must print the hand-checked answer. +//! +//! Before versions, `Map.remove` wrote into the map it was given, so a +//! caller that kept that map saw the key disappear, and `Map.set` copied +//! the whole table on every call whose receiver it could not prove unique +//! (an answer module's state field, for one). + +#![cfg(feature = "wasm")] + +#[path = "support/aver_cmd.rs"] +mod aver_cmd; + +use aver_cmd::{cleanup, format_output, temp_module}; + +use std::path::PathBuf; +use std::process::Command; + +fn run_cli(prefix: &str, source: &str, extra_args: &[&str]) -> String { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = temp_module(prefix, source); + let out = Command::new(env!("CARGO_BIN_EXE_aver")) + .current_dir(&repo_root) + .arg("run") + .arg(&path) + .args(extra_args) + .output() + .expect("expected `aver run` to execute"); + cleanup(&path); + assert!( + out.status.success(), + "{prefix} run {extra_args:?} failed:\n{}", + format_output(&out) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +fn assert_vm_and_wasm_gc(name: &str, source: &str, expected: &str) { + let vm = run_cli(name, source, &[]); + assert_eq!( + vm, expected, + "{name}: the VM diverged from the checked answer" + ); + let wasm = run_cli(name, source, &["--wasm-gc"]); + assert_eq!( + wasm, expected, + "{name}: wasm-gc diverged from the checked answer — a version of a map \ + saw a write made to a later one" + ); +} + +#[test] +fn remove_leaves_the_map_it_was_given_alone() { + assert_vm_and_wasm_gc( + "map-versions-remove", + r#" +module Main + intent = "Remove and set from one map, then read all three." + effects [Console.print] + +fn both(m: Map) -> Tuple, Map> + ? "Two maps from one." + (Map.remove(m, 1), Map.set(m, 2, 20)) + +fn main() -> Unit + ! [Console.print] + base = Map.set(Map.set({}, 1, 10), 3, 30) + pair = both(base) + match pair + (a, b) -> Console.print("{Map.len(base)} {Map.has(base, 1)} {Map.len(a)} {Map.has(a, 1)} {Map.len(b)} {Map.has(b, 1)} {Map.has(b, 2)} {Map.has(base, 2)}") +"#, + "2 true 1 false 3 true true false", + ); +} + +#[test] +fn every_version_reads_as_itself() { + assert_vm_and_wasm_gc( + "map-versions-all", + r#" +module Main + intent = "Every version of a map stays what it was when the next one is made from it." + effects [Console.print] + +fn fill(m: Map, from: Int, to: Int) -> Map + ? "Maps each key in from..to-1 to ten times itself." + match from >= to + true -> m + false -> fill(Map.set(m, from, from * 10), from + 1, to) + +fn total(m: Map, keys: List, acc: Int) -> Int + ? "Sums key * 1000 + value over the listed keys, and counts an absent one as -1." + match keys + [] -> acc + [k, ..rest] -> match Map.get(m, k) + Option.Some(v) -> total(m, rest, acc + k * 1000 + v) + Option.None -> total(m, rest, acc - 1) + +fn summary(m: Map) -> String + ? "Size, a checksum of the entries, and the first keys in order." + "{Map.len(m)}:{total(m, Map.keys(m), 0)}:{firstKeys(List.take(Map.keys(m), 4), "")}" + +fn firstKeys(keys: List, acc: String) -> String + ? "The keys, comma-separated." + match keys + [] -> acc + [k, ..rest] -> firstKeys(rest, "{acc},{k}") + +fn chain(m: Map, left: Int) -> Map + ? "Overwrites keys left down to 1 with 7." + match left <= 0 + true -> m + false -> chain(Map.set(m, left, 7), left - 1) + +fn removeAll(m: Map, keys: List) -> Map + ? "Removes every listed key." + match keys + [] -> m + [k, ..rest] -> removeAll(Map.remove(m, k), rest) + +fn main() -> Unit + ! [Console.print] + base = fill({}, 0, 40) + grown = fill(base, 40, 100) + updated = Map.set(base, 3, 999) + other = Map.set(base, 3, 111) + removed = removeAll(base, [0, 16, 32, 5, 21, 37, 99]) + long = chain(base, 30) + Console.print(summary(base)) + Console.print(summary(grown)) + Console.print(summary(updated)) + Console.print(summary(other)) + Console.print(summary(removed)) + Console.print(summary(long)) + Console.print(summary(base)) + Console.print("{base == fill({}, 0, 40)} {updated == other} {Map.set(base, 3, 30) == base} {removed == base} {Map.has(removed, 16)} {Map.has(base, 16)}") +"#, + "40:787800:,0,1,2,3\n\ + 100:4999500:,0,1,2,3\n\ + 40:788769:,0,1,2,3\n\ + 40:787881:,0,1,2,3\n\ + 34:675690:,1,2,3,4\n\ + 40:783360:,0,1,2,3\n\ + 40:787800:,0,1,2,3\n\ + true false true false false true", + ); +} + +/// The printed WAT of one `aver compile --target wasm-gc` of `source`. +fn wat_of(source: &str) -> String { + let mut items = aver::source::parse_source(source).expect("parse"); + let neutral_policy = aver::ir::NeutralAllocPolicy; + let result = aver::ir::pipeline::run( + &mut items, + aver::ir::PipelineConfig { + typecheck: Some(aver::ir::TypecheckMode::Full { base_dir: None }), + alloc_policy: Some(&neutral_policy), + run_interp_lower: false, + run_buffer_build: false, + run_chars_fusion: false, + run_list_build: false, + ..Default::default() + }, + ); + let tc = result.typecheck.as_ref().expect("typecheck requested"); + assert!(tc.errors.is_empty(), "typecheck failed: {:?}", tc.errors); + let bytes = aver::codegen::wasm_gc::compile_to_wasm_gc(&items, result.analysis.as_ref()) + .expect("wasm-gc compile"); + wasmprinter::print_bytes(&bytes).expect("print wat") +} + +/// An answer module's state reaches `Map.set` as a field of a record the +/// function was handed, which no ownership fact covers. The insert must +/// still not copy the table: its only array allocations are the grow's, and +/// it never copies an array. +#[test] +fn set_on_a_record_field_does_not_copy_the_table() { + let wat = wat_of( + r#" +module Main + intent = "A state record whose map every request updates." + effects [Console.print] + +record State + counts: Map + +fn bump(state: State, key: Int) -> State + ? "One more for this key." + next = Option.withDefault(Map.get(state.counts, key), 0) + 1 + State(counts = Map.set(state.counts, key, next)) + +verify bump + bump(State(counts = {}), 7) => State(counts = {7 => 1}) + +fn main() -> Unit + ! [Console.print] + Console.print("{Map.len(bump(State(counts = {}), 1).counts)}") +"#, + ); + let start = wat + .find("(func $\"Map.set Map") + .unwrap_or_else(|| panic!("no named Map.set helper in:\n{wat}")); + let body = &wat[start..]; + let body = &body[..body[1..] + .find("\n (func") + .map_or(body.len(), |end| end + 1)]; + assert!( + !body.contains("array.copy"), + "Map.set copies an array:\n{body}" + ); + assert_eq!( + body.matches("array.new_default").count(), + 3, + "Map.set allocates arrays outside its grow:\n{body}" + ); +} From a5434c3b983ee607159983e116377cdaa726408d Mon Sep 17 00:00:00 2001 From: jasisz Date: Fri, 25 Sep 2026 04:17:57 +0200 Subject: [PATCH 2/4] ci: run the map versions spec in the wasm-gc lane The spec is gated on the wasm feature, so the native lanes build it with no tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39608220c..8cfc76e63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -320,6 +320,7 @@ jobs: wasm_gc_effect_arg_overflow_regression \ wasm_gc_games_differential_mir \ wasm_gc_handler_carrier \ + wasm_gc_map_versions_spec \ wasm_gc_optimize_trunc_sat \ wasm_gc_packed_sequence \ wasm_gc_perslot_int_unboxing_differential \ From 9813887462391dbe08ed75228e924343e224f386 Mon Sep 17 00:00:00 2001 From: jasisz Date: Fri, 25 Sep 2026 04:29:52 +0200 Subject: [PATCH 3/4] wasm-gc: test a map's diff inline before calling reroot Every map helper called reroot on entry, and wasmtime does not inline the call, so map_lookup ran about 10% slower than before. Testing the diff field at the call site brings it within a few percent. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/codegen/wasm_gc/maps.rs | 22 +++++++++++----------- src/codegen/wasm_gc/maps/versions.rs | 15 +++++++++++++-- src/codegen/wasm_gc/module.rs | 2 +- src/codegen/wasm_gc/wasip2_http.rs | 7 ++++++- src/codegen/wasm_gc/wasip2_http_handler.rs | 7 ++++++- 5 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/codegen/wasm_gc/maps.rs b/src/codegen/wasm_gc/maps.rs index 040a82f96..f745921b6 100644 --- a/src/codegen/wasm_gc/maps.rs +++ b/src/codegen/wasm_gc/maps.rs @@ -1596,7 +1596,7 @@ fn emit_map_set( (1, map_ref), // 17: next version ]); - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); emit_insert_prologue(&mut f, slots); // query_hash = hash(k); idx = query_hash & mask; home = idx @@ -2047,7 +2047,7 @@ fn emit_map_get( ]); let _ = k_val; // cap, mask, keys, values - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2193,7 +2193,7 @@ fn emit_map_get_or_default( ]); // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2336,7 +2336,7 @@ fn emit_map_get_pair( (1, ValType::I32), // 10: query_hash ]); - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -2838,7 +2838,7 @@ fn emit_map_order_slots( // params: 0=map. locals: 1=keys, 2=indices, 3=count, 4=cap, // 5=slot, 6=used, 7=start, 8=end, 9=tmp. let mut f = Function::new([(1, keys_ref), (1, order_ref), (7, ValType::I32)]); - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3036,7 +3036,7 @@ fn emit_map_walk_keys_to_list( (1, list_ref), ]); // keys = map.keys - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3122,7 +3122,7 @@ fn emit_map_walk_values_to_list( (1, ValType::I32), (1, list_ref), ]); - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3249,7 +3249,7 @@ fn emit_map_eq( f.instruction(&Instruction::Return); f.instruction(&Instruction::End); // cap = a.cap; keys_a = a.keys; values_a = a.values; i = 0 - emit_reroot_local(&mut f, reroot_fn, 2); + emit_reroot_local(&mut f, slots.map, reroot_fn, 2); f.instruction(&Instruction::LocalGet(2)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3475,7 +3475,7 @@ fn emit_map_hash( f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::RefCastNonNull(map_heap)); f.instruction(&Instruction::LocalSet(1)); - emit_reroot_local(&mut f, reroot_fn, 1); + emit_reroot_local(&mut f, slots.map, reroot_fn, 1); f.instruction(&Instruction::I32Const(0)); f.instruction(&Instruction::LocalSet(7)); f.instruction(&Instruction::LocalGet(1)); @@ -3641,7 +3641,7 @@ fn emit_map_remove( }; // cap = map.cap; mask = cap - 1; keys = map.keys; values = map.values - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, @@ -3935,7 +3935,7 @@ fn emit_map_entries( (1, ValType::I32), (1, lt_ref), ]); - emit_reroot_local(&mut f, reroot_fn, 0); + emit_reroot_local(&mut f, slots.map, reroot_fn, 0); f.instruction(&Instruction::LocalGet(0)); f.instruction(&Instruction::StructGet { struct_type_index: slots.map, diff --git a/src/codegen/wasm_gc/maps/versions.rs b/src/codegen/wasm_gc/maps/versions.rs index 2948ae8c3..a997d8bd2 100644 --- a/src/codegen/wasm_gc/maps/versions.rs +++ b/src/codegen/wasm_gc/maps/versions.rs @@ -61,16 +61,27 @@ pub(in crate::codegen::wasm_gc) fn emit_no_diff(f: &mut Function, slots: MapSlot f.instruction(&Instruction::RefNull(HeapType::Concrete(slots.diff))); } -/// `local.get map; call reroot; local.set map`: make the version in -/// `map_local` the current one before its arrays are read. +/// Make the version in `map_local` the current one before its arrays are +/// read. The null-diff test is inline, so a map that is already current, +/// which is nearly every map, pays a field read and no call. pub(in crate::codegen::wasm_gc) fn emit_reroot_local( f: &mut Function, + map_type: u32, reroot_fn: u32, map_local: u32, ) { + f.instruction(&Instruction::LocalGet(map_local)); + f.instruction(&Instruction::StructGet { + struct_type_index: map_type, + field_index: DIFF_FIELD, + }); + f.instruction(&Instruction::RefIsNull); + f.instruction(&Instruction::I32Eqz); + f.instruction(&Instruction::If(BlockType::Empty)); f.instruction(&Instruction::LocalGet(map_local)); f.instruction(&Instruction::Call(reroot_fn)); f.instruction(&Instruction::LocalSet(map_local)); + f.instruction(&Instruction::End); } /// `reroot(map) -> map`: make `map` the version that owns its arrays' diff --git a/src/codegen/wasm_gc/module.rs b/src/codegen/wasm_gc/module.rs index 5124496cd..14ca0f3b1 100644 --- a/src/codegen/wasm_gc/module.rs +++ b/src/codegen/wasm_gc/module.rs @@ -10970,7 +10970,7 @@ fn emit_handler_wrapper( "aver_http_handle wrapper requires the Map> helpers".into(), ))? .reroot; - super::maps::emit_reroot_local(&mut f, headers_reroot, 9); + super::maps::emit_reroot_local(&mut f, map_slots.map, headers_reroot, 9); // Read map cap + arrays into iteration slots. f.instruction(&Instruction::LocalGet(9)); diff --git a/src/codegen/wasm_gc/wasip2_http.rs b/src/codegen/wasm_gc/wasip2_http.rs index effaf76ac..9ce44ddb2 100644 --- a/src/codegen/wasm_gc/wasip2_http.rs +++ b/src/codegen/wasm_gc/wasip2_http.rs @@ -777,7 +777,12 @@ pub(super) fn emit_http_get(indices: &HttpGetIndices, h: &HttpGetHelperFns) -> F // but the dispatcher passes an empty map, so it's a no-op // beyond the cap-iter (~16k iterations of "is keys[i] null? // yes, skip"). Acceptable for v1 PoC. - super::maps::emit_reroot_local(&mut f, indices.headers_map_reroot_fn, p_headers); + super::maps::emit_reroot_local( + &mut f, + indices.headers_map_type_idx, + indices.headers_map_reroot_fn, + p_headers, + ); f.instruction(&Instruction::LocalGet(p_headers)); f.instruction(&Instruction::StructGet { struct_type_index: indices.headers_map_type_idx, diff --git a/src/codegen/wasm_gc/wasip2_http_handler.rs b/src/codegen/wasm_gc/wasip2_http_handler.rs index 8df1b878c..f890703e8 100644 --- a/src/codegen/wasm_gc/wasip2_http_handler.rs +++ b/src/codegen/wasm_gc/wasip2_http_handler.rs @@ -1174,7 +1174,12 @@ pub(super) fn emit_aver_http_handle( } // Walk response headers map → fields.append. - super::maps::emit_reroot_local(&mut f, indices.headers_map_reroot_fn, l_resp_headers_map); + super::maps::emit_reroot_local( + &mut f, + map_idx, + indices.headers_map_reroot_fn, + l_resp_headers_map, + ); f.instruction(&Instruction::LocalGet(l_resp_headers_map)); f.instruction(&Instruction::StructGet { struct_type_index: map_idx, From ded12e9908b602b4a9438a8a3faf7810b82b2cb5 Mon Sep 17 00:00:00 2001 From: jasisz Date: Fri, 25 Sep 2026 04:30:04 +0200 Subject: [PATCH 4/4] changelog: quote the release-build timings for the map fix Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44b25284c..e2a180eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,7 @@ The generated loop is now written from the program's source alone, and the manif ### Fixed -- **wasm-gc: `Map.set` no longer copies the map, and `Map.remove` no longer changes the map it was given.** `set` copied every bucket unless the compiler could prove the map had no other holder, which it cannot for a map held in a record field such as an answer module's state. A 100 000-entry state map served 2000 `Map.set` requests in 558 s under `aver run --wasm-gc` and 1.9 s on Node 26. `remove` wrote into the map it was given, so a caller that still held that map saw the key gone. Both now write into the map's arrays in place and return a new version. The version they were given stays valid, because a record of what the write replaced is kept with it. The same run now takes 7 s under `aver run --wasm-gc`, nearly all of it startup and building the map, and 0.3 s on Node. Reading an older version again costs one step for each write made since. +- **wasm-gc: `Map.set` no longer copies the map, and `Map.remove` no longer changes the map it was given.** `set` copied every bucket unless the compiler could prove the map had no other holder, which it cannot for a map held in a record field such as an answer module's state. A 100 000-entry state map served 2000 `Map.set` requests in 11 s under `aver run --wasm-gc` and 1.9 s on Node 26. `remove` wrote into the map it was given, so a caller that still held that map saw the key gone. Both now write into the map's arrays in place and return a new version. The version they were given stays valid, because a record of what the write replaced is kept with it. The same run now takes 0.4 s under `aver run --wasm-gc` and 0.3 s on Node. Reading an older version again costs one step for each write made since. - **A wait over sockets and jobs keeps watching its sockets after a job outside its set settles.** The wake from that job used to end the socket poll, and the wait then slept out the rest of its timeout on the job engine alone, missing sockets that became ready meanwhile and never reporting them. The VM, generated Rust and the wasm-gc native host now share one wait loop that polls the whole set again. - **A handle whose slot the engine has forgotten answers `work: unknown job` on the VM**, as it already did elsewhere, instead of claiming another job kind started it. Which kind began a job is now kept in the job's own slot, so nothing a job kind keeps grows with the number of jobs it starts. - **The VM runs a function whose bytecode is larger than 32 KiB.** Jump offsets were sixteen bits and a longer forward jump wrapped into a backward one, which crashed `aver verify` on large generated trace laws.