From 17363b000ccd259ffcf91bd77b5b97a0af61ad35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 06:24:33 +0200 Subject: [PATCH 1/7] perf(codegen): keep the element-shape clone's accumulator and counter in native domains The fast clone now redirects the accumulator into a promotable f64 alloca and defers the counter's double storage to an i32 slot the Update lowering advances alone, publishing both at a side-exit trampoline and on the fall-through exit. --- .../src/stmt/element_shape_loop.rs | 18 +- .../src/stmt/element_shape_native.rs | 246 ++++++++++++++++++ crates/perry-codegen/src/stmt/mod.rs | 1 + 3 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 crates/perry-codegen/src/stmt/element_shape_native.rs diff --git a/crates/perry-codegen/src/stmt/element_shape_loop.rs b/crates/perry-codegen/src/stmt/element_shape_loop.rs index 47e54c7c46..6f657bb8a9 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop.rs @@ -238,6 +238,8 @@ impl MatchedIndex { #[derive(Debug)] struct ElementShapeVersionedLoop { counter_id: u32, + /// The counter's integer-literal start, in `0..=i32::MAX`. + counter_start: i64, bound: ElementShapeLoopBound, array_id: u32, identity: ElementShapeIdentity, @@ -1314,6 +1316,7 @@ fn match_element_shape_versioned_loop( Some(ElementShapeVersionedLoop { counter_id, + counter_start: start, bound, array_id, identity, @@ -1711,6 +1714,18 @@ pub(super) fn lower_element_shape_versioned_for( let scope_id = ctx.next_loop_proof_scope_id(); let fast_scan_start = ctx.func.num_blocks(); ctx.current_block = fast_pre_idx; + // The accumulator as an f64 and the counter as an i32 for the clone's + // duration (`stmt/element_shape_native.rs`). Entered after + // `fast_scan_start`, so its side-exit trampoline is inside the block range + // the call-free scan below covers. + let native = super::element_shape_native::NativeLoopDomains::enter( + ctx, + matched.counter_id, + matched.counter_start, + matched.accumulator_id, + &accumulator, + &slow_pre_label, + ); ctx.element_shape_loop_facts .push(crate::expr::ElementShapeLoopFact { array_local_id: matched.array_id, @@ -1722,7 +1737,7 @@ pub(super) fn lower_element_shape_versioned_for( class_name: report_class, elements_base: guard.elements_base, expected_shape_id: guard.expected_shape_id, - side_exit_label: slow_pre_label.clone(), + side_exit_label: native.side_exit_label().to_string(), statically_layout_proven, fields, synthesized_body: matched.fast_body.is_some(), @@ -1740,6 +1755,7 @@ pub(super) fn lower_element_shape_versioned_for( ); ctx.element_shape_loop_facts .retain(|fact| fact.scope_id != scope_id); + native.finish(ctx, &merge_label); lowered?; if !ctx.block().is_terminated() { ctx.block().br(&merge_label); diff --git a/crates/perry-codegen/src/stmt/element_shape_native.rs b/crates/perry-codegen/src/stmt/element_shape_native.rs new file mode 100644 index 0000000000..a406800e64 --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_native.rs @@ -0,0 +1,246 @@ +//! The element-shape fast clone's loop-carried scalars in their native +//! machine domains: the accumulator as an `f64`, the counter as an `i32`. +//! +//! ## What this removes +//! +//! Before this module the `repeat` clone (`sum += rows[7].id`) spent ~13 CPU +//! cycles per iteration on 16 instructions. Not instruction count: two +//! loop-carried chains crossed the integer/float register boundary every +//! iteration. +//! +//! * **The accumulator.** An `any`-typed `sum` lives in a precise GC root slot, +//! and every reload of a root slot passes through the RS4GC launder +//! (`function/precise_roots.rs`'s `ROOT_RELOAD_LAUNDER`, an `asm` identity +//! LLVM cannot see through). mem2reg therefore promoted the slot as NaN-box +//! `i64` bits, and the chain was `x25 → fmov → fadd → fmov → x25`. +//! * **The counter.** A counter the clone never uses as an index (the +//! constant-index and carried forms) has no i32 slot, so it stayed a double +//! (`fadd d8, d8, #1.0`) compared against `count` — itself a root slot, +//! reloaded through the same launder and re-transferred to an FP register on +//! every iteration, although the preheader had already materialized it as an +//! i32. +//! +//! ## What replaces it +//! +//! Both scalars move into plain promotable allocas for the fast clone's +//! lowering, exactly the redirects the packed clones already use +//! (`FnCtx::numeric_accumulator_f64_slots`, +//! `FnCtx::deferred_integer_update_accumulators`): +//! +//! * the accumulator's live value is an `alloca double` seeded in the fast +//! preheader with the value the deref block just tag-tested as a Number, so +//! every in-clone read and write is a `double` and mem2reg makes it an FP phi; +//! * the counter gets an i32 slot (reused if it already owns a parallel one, +//! clone-private otherwise) that the `Update` lowering advances ALONE, so the +//! precomputed i32 trip count turns the condition into `icmp slt i32`. +//! +//! The real slots are written back at the two places the clone can leave: +//! a side-exit trampoline every residual check branches to instead of the slow +//! preheader, and a block on the fall-through exit. +//! +//! ## Why this changes no observable behaviour +//! +//! * **Commit ordering (#10185) is untouched.** The redirected store happens at +//! exactly the point the root-slot store used to, and the matcher already +//! put that point after every side exit of the iteration (the K-statement +//! fold, the carried commit last). At any side exit the f64 alloca therefore +//! holds the value the iteration was entered with, the i32 counter holds the +//! iteration's own index (the `Update` runs after the body), and the +//! trampoline publishes exactly the state the slow clone must re-run the +//! iteration from. +//! * **Call-freeness is untouched.** The seed, the trampoline and the +//! write-back are plain loads, stores and one `sitofp`; the trampoline is +//! created inside the scanned block range, so the post-emission +//! `contains_gc_unsafe_call` scan still sees it. No poll is emitted in these +//! clones (`emit_gc_loop_safepoint`), so no collection observes the stale +//! root slot, and a stale slot holds a Number, which a scan treats as data. +//! * **JS `+`.** The redirect admits no write the clone did not already lower +//! as a bare `fadd`: the accumulator is the fact's `numeric_accumulator`, +//! which `is_numeric_expr` already trusted as a raw double inside this clone. +//! Keeping that same double in a register instead of a stack slot is the +//! same IEEE arithmetic on the same operands in the same order, so `-0`, NaN +//! and overflow to Infinity are bit-identical. +//! * **The i32 counter.** The trip count is always an i32 in the fast clone — +//! a literal in `0..=i32::MAX`, `arr.length`, or `materialize_loop_i32`'s +//! integral `0..=i32::MAX` — and the start is an integer literal in the same +//! range, so `i < bound <= i32::MAX` before every `add i32 1`: no wrap. A +//! fractional, NaN, negative or out-of-range `count` never reaches the clone. +//! +//! Every exclusion below declines ONE redirect and leaves that scalar exactly +//! as it was, never the clone. + +use crate::expr::FnCtx; +use crate::types::{DOUBLE, I32}; + +/// The scalars one fast clone moved into native storage, and where the clone +/// must publish them back. +pub(super) struct NativeLoopDomains { + /// `(local, f64 alloca, real slot)`. + accumulator: Option<(u32, String, String)>, + /// The counter's deferred double storage. + counter: Option, + /// The label residual checks branch to: the write-back trampoline, or the + /// slow preheader itself when nothing was redirected. + side_exit: String, +} + +struct DeferredCounter { + id: u32, + i32_slot: String, + double_slot: String, + /// The slot was minted for this clone and must be unregistered after it, + /// so the slow clone and the code after the loop see the counter exactly + /// as they did before. + private: bool, +} + +/// Can `id` carry the unboxed accumulator redirect? Mirrors the packed clones' +/// admission (`stable_packed_accumulator::collect_numeric_accumulators`) minus +/// the numeric fixpoint, which the element-shape matcher already discharged. +fn accumulator_is_redirectable(ctx: &FnCtx<'_>, id: u32) -> bool { + ctx.locals.contains_key(&id) + && !ctx.boxed_vars.contains(&id) + && !ctx.closure_captures.contains_key(&id) + && !ctx.module_globals.contains_key(&id) + // An i32 (parallel or canonical) or Str representation has storage the + // redirect would leave stale; a POD or scalar-replaced local is not a + // single slot at all. + && !ctx.i32_counter_slots.contains_key(&id) + && !ctx.local_slot_reps.contains_key(&id) + && !ctx.pod_records.contains_key(&id) + && !ctx.scalar_replaced.contains_key(&id) + // An enclosing clone already redirected it; its scope owns the alloca + // and the write-back, and overwriting the entry would strand both. + && !ctx.numeric_accumulator_f64_slots.contains_key(&id) +} + +impl NativeLoopDomains { + /// Emit the seeds into the current block (the fast preheader) and register + /// both redirects for the fast clone's lowering. + /// + /// `accumulator_value` is the value the deref block tag-tested as a Number; + /// the deref block dominates the fast preheader and nothing is stored in + /// between, so it is the slot's current value. + pub(super) fn enter( + ctx: &mut FnCtx<'_>, + counter_id: u32, + counter_start: i64, + accumulator_id: u32, + accumulator_value: &str, + slow_pre_label: &str, + ) -> Self { + let accumulator = accumulator_is_redirectable(ctx, accumulator_id).then(|| { + let real_slot = ctx.locals[&accumulator_id].clone(); + let alloca = ctx.func.alloca_entry(DOUBLE); + ctx.block().store(DOUBLE, accumulator_value, &alloca); + ctx.numeric_accumulator_f64_slots + .insert(accumulator_id, alloca.clone()); + (accumulator_id, alloca, real_slot) + }); + + let counter = Self::defer_counter(ctx, counter_id, counter_start); + + let side_exit = if accumulator.is_none() && counter.is_none() { + slow_pre_label.to_string() + } else { + let tramp_idx = ctx.new_block("element_shape.loop.side_exit"); + let saved = ctx.current_block; + ctx.current_block = tramp_idx; + emit_write_back(ctx, accumulator.as_ref(), counter.as_ref()); + ctx.block().br(slow_pre_label); + ctx.current_block = saved; + ctx.block_label(tramp_idx) + }; + + Self { + accumulator, + counter, + side_exit, + } + } + + fn defer_counter(ctx: &mut FnCtx<'_>, id: u32, start: i64) -> Option { + // A canonical-i32 counter has no double storage to defer — its `Update` + // is already one `add i32`. + if crate::expr::canonical_local_i32_slot(ctx, id).is_some() + || ctx.boxed_vars.contains(&id) + || ctx.closure_captures.contains_key(&id) + || ctx.module_globals.contains_key(&id) + || ctx.unsigned_i32_locals.contains(&id) + || ctx.deferred_integer_update_accumulators.contains(&id) + || !(0..=i64::from(i32::MAX)).contains(&start) + { + return None; + } + let double_slot = ctx.locals.get(&id)?.clone(); + let (i32_slot, private) = match ctx.i32_counter_slots.get(&id) { + // A Let-site parallel slot: every write mirrors it, so it already + // holds the counter's value. + Some(slot) => (slot.clone(), false), + None => { + let slot = ctx.func.alloca_entry(I32); + // The init was lowered before the clone was chosen and nothing + // wrote the counter since, so its value is the literal start. + ctx.block().store(I32, &start.to_string(), &slot); + ctx.i32_counter_slots.insert(id, slot.clone()); + (slot, true) + } + }; + ctx.deferred_integer_update_accumulators.insert(id); + Some(DeferredCounter { + id, + i32_slot, + double_slot, + private, + }) + } + + /// The label every residual check of the fast clone must branch to. + pub(super) fn side_exit_label(&self) -> &str { + &self.side_exit + } + + /// Publish the live values on the fall-through exit and end both + /// redirects. Must run right after the fast clone is lowered and BEFORE + /// the slow clone is, which reads and writes the real slots. + pub(super) fn finish(self, ctx: &mut FnCtx<'_>, merge_label: &str) { + let redirected = self.accumulator.is_some() || self.counter.is_some(); + if redirected && !ctx.block().is_terminated() { + let commit_idx = ctx.new_block("element_shape.loop.fast.write_back"); + let commit_label = ctx.block_label(commit_idx); + ctx.block().br(&commit_label); + ctx.current_block = commit_idx; + emit_write_back(ctx, self.accumulator.as_ref(), self.counter.as_ref()); + ctx.block().br(merge_label); + } + if let Some((id, _, _)) = &self.accumulator { + ctx.numeric_accumulator_f64_slots.remove(id); + } + if let Some(counter) = &self.counter { + ctx.deferred_integer_update_accumulators.remove(&counter.id); + if counter.private { + ctx.i32_counter_slots.remove(&counter.id); + } + } + } +} + +/// The write-back both exits share. A genuine double's bits are its NaN-box, so +/// the accumulator needs no conversion and — carrying no heap edge — no +/// barrier; the counter is an exact integer in `0..=i32::MAX`. +fn emit_write_back( + ctx: &mut FnCtx<'_>, + accumulator: Option<&(u32, String, String)>, + counter: Option<&DeferredCounter>, +) { + let blk = ctx.block(); + if let Some((_, alloca, real_slot)) = accumulator { + let value = blk.load(DOUBLE, alloca); + blk.store(DOUBLE, &value, real_slot); + } + if let Some(counter) = counter { + let value = blk.load(I32, &counter.i32_slot); + let as_double = blk.sitofp(I32, &value, DOUBLE); + blk.store(DOUBLE, &as_double, &counter.double_slot); + } +} diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 73e8ecaf05..18e645070f 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -22,6 +22,7 @@ mod class_field_loop_tests; mod counter_range; mod element_shape_carried; mod element_shape_loop; +mod element_shape_native; #[cfg(test)] mod element_shape_loop_tests; mod if_stmt; From 7c5e7c3711444109bb0801a72bb6aa3511a2794f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:38:39 +0200 Subject: [PATCH 2/7] test(codegen): pin the element-shape clone's native accumulator and counter IR census for the f64 accumulator redirect, the i32 counter, the side-exit trampoline and the fall-through write-back; the carried-commit and side-exit helpers count the trampoline spelling of an exit. --- .../stmt/element_shape_fields_random_tests.rs | 39 ++- .../src/stmt/element_shape_native_tests.rs | 310 ++++++++++++++++++ 2 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 crates/perry-codegen/src/stmt/element_shape_native_tests.rs diff --git a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs index 5976551ff8..3359c4c4b3 100644 --- a/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs +++ b/crates/perry-codegen/src/stmt/element_shape_fields_random_tests.rs @@ -208,10 +208,26 @@ fn fields_body() -> Vec { } /// How many side exits the clone can take per iteration — one branch per -/// residual / tag test that leaves for the slow preheader. +/// residual / tag test that leaves the clone. +/// +/// A side exit targets the slow preheader directly, or — once the clone keeps +/// its accumulator or counter in native storage (`stmt/element_shape_native.rs`) +/// — the write-back trampoline that publishes them and then enters the slow +/// preheader. Both spellings are one exit; counting only the first would make +/// every assertion below read zero for a clone that exits through the second. fn side_exit_count(fast: &str) -> usize { fast.matches("label %element_shape.loop.slow.preheader") .count() + + fast.matches("label %element_shape.loop.side_exit").count() +} + +/// The registers `block` defines with a `sitofp i32 … to double`. +fn sitofp_results(block: &str) -> Vec<&str> { + block + .lines() + .filter(|l| l.contains(" = sitofp i32 ")) + .filter_map(|l| l.trim_start().split(" = ").next()) + .collect() } /// The LAST block in `fast` whose label starts with `prefix`, body only. @@ -311,8 +327,21 @@ fn the_carried_write_back_follows_every_side_exit() { iteration, and every later index is silently wrong. \ final block:\n{committed}\nfull clone:\n{fast}" ); + // The accumulator's store is ALSO a `store double` in this block now that + // it lands in the clone's f64 alloca (`stmt/element_shape_native.rs`), so + // the publication is counted by what it stores: the carried i32 converted + // for its real slot. + let published = sitofp_results(committed); assert_eq!( - committed.matches("store double").count(), + published.len(), + 1, + "the carried value must be converted for publication exactly once per \ + iteration; final block:\n{committed}" + ); + assert_eq!( + committed + .matches(&format!("store double {}, ", published[0])) + .count(), 1, "the carried value must be published exactly once per iteration; \ final block:\n{committed}" @@ -652,3 +681,9 @@ fn a_class_typed_array_declines_the_string_length_read() { "the class-keyed arm reads raw doubles; there is no tag to test" ); } + +/// The accumulator and counter in their native domains +/// (`stmt/element_shape_native.rs`) — a child module so it reuses this file's +/// `random`/`fields` fixtures and exit-counting helpers. +#[path = "element_shape_native_tests.rs"] +mod native_domains; diff --git a/crates/perry-codegen/src/stmt/element_shape_native_tests.rs b/crates/perry-codegen/src/stmt/element_shape_native_tests.rs new file mode 100644 index 0000000000..57217bdc5a --- /dev/null +++ b/crates/perry-codegen/src/stmt/element_shape_native_tests.rs @@ -0,0 +1,310 @@ +//! The element-shape fast clone's accumulator as an `f64` and its counter as an +//! `i32` (`stmt/element_shape_native.rs`). +//! +//! A child module of `element_shape_fields_random_tests`, so `use super::*` +//! brings that file's `random`/`fields` fixtures, the shape-keyed file's +//! untyped-receiver fixtures and `element_shape_loop_tests`'s slicing helpers. +//! +//! Every assertion here names storage by what the emitted IR itself says it is +//! — the fast preheader's seeds, the deref block's tag-tested reload — rather +//! than by register number, so a renumbering cannot make one vacuous. Each one +//! fails on the pre-change clone, which kept `sum` in its GC root slot (a +//! `ptr addrspace(1)` alloca reloaded through the RS4GC launder every +//! iteration) and the `repeat` counter as a double compared with `fcmp`. + +use super::*; + +/// `for (let i = 0; i < count; i++) sum += rows[7].id` with an untyped `sum` — +/// the benchmark's `repeat` mode, where both scalars were off-domain. +fn repeat_ir() -> String { + emit(&untyped_param_module( + Type::Any, + Type::Any, + vec![untyped_accumulate(untyped_elem_field( + Expr::Integer(7), + "v", + ))], + )) +} + +/// The first `store , ptr ` in `block` whose type is `ty`, +/// as `(value, slot)`. +fn first_store<'a>(block: &'a str, ty: &str) -> (&'a str, &'a str) { + block + .lines() + .find_map(|l| { + let rest = l.trim_start().strip_prefix(&format!("store {ty} "))?; + rest.split_once(", ptr ") + }) + .unwrap_or_else(|| panic!("no `store {ty}` in:\n{block}")) +} + +/// The root alloca a laundered reload `reg` was read from +/// (`.rs4p = load ptr addrspace(1), ptr `). +fn root_slot_of_reload<'a>(ir: &'a str, reg: &str) -> &'a str { + let needle = format!("{reg}.rs4p = load ptr addrspace(1), ptr "); + let at = ir + .find(&needle) + .unwrap_or_else(|| panic!("`{reg}` is not a root-slot reload")); + ir[at + needle.len()..].lines().next().unwrap().trim() +} + +/// Is `block`'s terminator an unconditional branch to the block `target` +/// names (`label %`, without its numeric suffix)? +fn terminator_targets(block: &str, target: &str) -> bool { + block + .lines() + .last() + .and_then(|l| l.trim().strip_prefix(&format!("br {target}."))) + .is_some_and(|suffix| suffix.chars().all(|c| c.is_ascii_digit())) +} + +/// `(accumulator alloca, accumulator root slot, counter i32 slot)` for the +/// repeat clone, read off the fast preheader's two seeds. +fn repeat_storage(ir: &str) -> (String, String, String) { + let pre = block_slice(ir, "element_shape.loop.fast.preheader"); + let (acc_value, acc_alloca) = first_store(pre, "double"); + let (start, counter_slot) = first_store(pre, "i32"); + assert_eq!( + start, "0", + "the counter's i32 slot must be seeded with the loop's literal start; \ + fast preheader:\n{pre}" + ); + let root = root_slot_of_reload(ir, acc_value); + ( + acc_alloca.to_string(), + root.to_string(), + counter_slot.to_string(), + ) +} + +#[test] +fn the_accumulator_never_touches_its_root_slot_inside_the_clone() { + let ir = repeat_ir(); + assert_shape_keyed_clone(&ir, "repeat with an untyped accumulator"); + let (acc_alloca, acc_root, _) = repeat_storage(&ir); + let fast = fast_clone_slice(&ir); + // The seed is the value the deref block tag-tested as a Number — the + // soundness of every raw `fadd` below rests on that being the same value. + let deref = block_slice(&ir, "element_shape.loop.preheader.deref"); + let seeded = first_store( + block_slice(&ir, "element_shape.loop.fast.preheader"), + "double", + ) + .0; + assert!( + deref.contains(&format!("{seeded} = bitcast i64 {seeded}.rs4o to double")) + && deref.contains(&format!("bitcast double {seeded} to i64")), + "the f64 alloca must be seeded with the reload the deref block \ + Number-tested; deref:\n{deref}" + ); + assert!( + !fast.contains("addrspace(1)") && !fast.contains("asm \"\""), + "no root slot may be loaded or stored inside the clone: every reload \ + passes through the RS4GC launder, which pins the loop-carried value to \ + a GPR and costs two GPR<->FPR transfers per iteration; emitted:\n{fast}" + ); + assert_eq!( + fast.matches(&format!("load double, ptr {acc_alloca}")) + .count(), + 1, + "the accumulator must be read from its f64 alloca once per iteration; \ + emitted:\n{fast}" + ); + assert_eq!( + fast.lines() + .filter(|l| { + l.trim_start().starts_with("store double ") + && l.ends_with(&format!(", ptr {acc_alloca}")) + }) + .count(), + 1, + "the accumulator must be committed to its f64 alloca exactly once per \ + iteration; emitted:\n{fast}" + ); + assert!( + !fast.contains(&format!("ptr {acc_root}")), + "the root slot `{acc_root}` must not appear inside the clone; \ + emitted:\n{fast}" + ); + assert_eq!( + fast.matches("fadd double").count(), + 1, + "exactly one IEEE add per iteration — the accumulator's; a second one \ + is the double counter this change retires; emitted:\n{fast}" + ); +} + +#[test] +fn the_repeat_counter_is_an_i32_against_the_materialized_bound() { + let ir = repeat_ir(); + let (_, _, counter_slot) = repeat_storage(&ir); + let cond = block_slice(&ir, "for.element_shape_fast.cond"); + assert!( + cond.contains(&format!("load i32, ptr {counter_slot}")) + && cond.contains("icmp slt i32") + && !cond.contains("fcmp") + && !cond.contains("addrspace(1)"), + "the condition must compare the i32 counter with the preheader's \ + materialized i32 bound — not reload `count` through its root slot and \ + `fcmp` a double counter against it; cond:\n{cond}" + ); + let update = block_slice(&ir, "for.element_shape_fast.update"); + assert!( + update.contains(&format!("load i32, ptr {counter_slot}")) + && update.contains("add i32") + && update.contains("store i32") + && !update.contains("store double") + && !update.contains("fadd"), + "`i++` must advance ONLY the i32 slot inside the clone; update:\n{update}" + ); + // The bound the i32 compare trusts is the validated materialization: a + // fractional, NaN, negative or out-of-range `count` branches to the slow + // clone before any of this is reachable. + for label in [ + "element_shape.loop.bound.range", + "element_shape.loop.bound.convert", + ] { + let block = block_slice(&ir, label); + assert!( + block.contains("label %element_shape.loop.slow.preheader"), + "`{label}` must still route an unrepresentable bound to the slow \ + clone; emitted:\n{block}" + ); + } +} + +#[test] +fn every_side_exit_publishes_both_scalars_before_the_slow_clone() { + let ir = repeat_ir(); + let (acc_alloca, acc_root, counter_slot) = repeat_storage(&ir); + let fast = fast_clone_slice(&ir); + assert_eq!( + fast.matches("label %element_shape.loop.slow.preheader") + .count(), + 0, + "no side exit may reach the slow clone without publishing the \ + redirected scalars first; emitted:\n{fast}" + ); + assert_eq!( + fast.matches("label %element_shape.loop.side_exit").count(), + 2, + "the residual check and the Number tag test must both leave through \ + the write-back trampoline; emitted:\n{fast}" + ); + for (label, successor) in [ + ( + "element_shape.loop.side_exit", + "label %element_shape.loop.slow.preheader", + ), + ( + "element_shape.loop.fast.write_back", + "label %element_shape.loop.merge", + ), + ] { + let block = block_slice(&ir, label); + let acc_load = block + .lines() + .position(|l| l.contains(&format!("load double, ptr {acc_alloca}"))); + let acc_publish = block.lines().position(|l| { + l.contains("store ptr addrspace(1)") && l.ends_with(&format!("ptr {acc_root}")) + }); + let counter_publish = block + .lines() + .position(|l| l.contains("store double %") && !l.contains("addrspace")); + assert!( + acc_load.is_some() + && acc_publish.is_some() + && block.contains(&format!("load i32, ptr {counter_slot}")) + && block.contains("sitofp i32") + && counter_publish.is_some() + && terminator_targets(block, successor) + && !block.contains("call "), + "`{label}` must publish the accumulator to its root slot `{acc_root}` \ + and the counter to its double slot, call-free, then branch to \ + `{successor}`; emitted:\n{block}" + ); + } + let exit = block_slice(&ir, "for.element_shape_fast.exit"); + assert!( + exit.contains("label %element_shape.loop.fast.write_back"), + "the fall-through exit must publish too; emitted:\n{exit}" + ); +} + +/// The carried index already had its own commit protocol (#10185). The +/// trampoline must publish the accumulator and the counter but NEVER the +/// carried binding: its real slot has to keep the previous iteration's commit, +/// which is the value the slow clone re-runs the recurrence from. +#[test] +fn the_trampoline_leaves_the_carried_binding_to_its_own_commit() { + let ir = emit(&access_module(random_body(true))); + assert_shape_keyed_clone(&ir, "random"); + let fast = fast_clone_slice(&ir); + let committed = last_block_with_prefix(&fast, "element_shape.number"); + let published = sitofp_results(committed); + assert_eq!( + published.len(), + 1, + "one carried commit; block:\n{committed}" + ); + let carried_slot = committed + .lines() + .find_map(|l| { + l.trim_start() + .strip_prefix(&format!("store double {}, ptr ", published[0])) + }) + .expect("the carried commit stores the converted value"); + let tramp = block_slice(&ir, "element_shape.loop.side_exit"); + assert!( + !tramp.contains(&format!("ptr {carried_slot}\n")) + && !tramp.trim_end().ends_with(&format!("ptr {carried_slot}")), + "the trampoline must not write the carried binding `{carried_slot}`: a \ + side exit happens AFTER this iteration's recurrence, so publishing it \ + would make the slow clone apply the recurrence twice; emitted:\n{tramp}" + ); + assert_eq!( + sitofp_results(tramp).len(), + 1, + "the trampoline converts exactly one i32 — the loop counter; \ + emitted:\n{tramp}" + ); + let cond = block_slice(&ir, "for.element_shape_fast.cond"); + assert!( + cond.contains("icmp slt i32") && !cond.contains("fcmp"), + "the carried form's counter must be an i32 too; cond:\n{cond}" + ); + assert!( + !fast.contains("addrspace(1)"), + "no root slot inside the carried clone; emitted:\n{fast}" + ); +} + +/// `fields`: the counter is already a canonical i32 (it indexes), so only the +/// accumulator is redirected — and all three folded adds stay in the double +/// domain. +#[test] +fn the_fields_clone_redirects_only_the_accumulator() { + let ir = emit(&access_module(fields_body())); + assert_shape_keyed_clone(&ir, "fields"); + let fast = fast_clone_slice(&ir); + assert!( + !fast.contains("addrspace(1)"), + "no root slot inside the fields clone; emitted:\n{fast}" + ); + assert_eq!(fast.matches("fadd double").count(), 3); + let tramp = block_slice(&ir, "element_shape.loop.side_exit"); + assert!( + tramp.contains("store ptr addrspace(1)") + && sitofp_results(tramp).is_empty() + && terminator_targets(tramp, "label %element_shape.loop.slow.preheader"), + "a canonical-i32 counter has no double storage to publish, so the \ + trampoline carries only the accumulator; emitted:\n{tramp}" + ); + assert_eq!( + side_exit_count(&fast), + fast.matches("label %element_shape.loop.side_exit").count(), + "every side exit of the fields clone must go through the trampoline; \ + emitted:\n{fast}" + ); +} From d13d5368fd159893d46948fbc87b617d70ae350c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:40:19 +0200 Subject: [PATCH 3/7] test(gap): observe the clone's accumulator and counter publications and IEEE edges --- test-files/test_gap_json_record_loop_clone.ts | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/test-files/test_gap_json_record_loop_clone.ts b/test-files/test_gap_json_record_loop_clone.ts index 363fb7a04b..b57b3bb962 100644 --- a/test-files/test_gap_json_record_loop_clone.ts +++ b/test-files/test_gap_json_record_loop_clone.ts @@ -568,3 +568,100 @@ console.log( "own-named-fields-again:", ownNamedFields(ownNames, 12, ownNames.length), ); + +// --------------------------------------------------------------------------- +// 10. The accumulator as an f64 and the counter as an i32 inside the clone. +// +// Both scalars now live in native storage for the clone's duration and are +// published to their real slots only at a side exit and at the loop's +// exit. Each case below observes one of those publications, or one of the +// IEEE edge cases the double-domain add must reproduce exactly. +// --------------------------------------------------------------------------- + +// `repeat` from an arbitrary starting accumulator. +function repeatFrom(rows: any, count: number, start: any): any { + let sum: any = start; + for (let i = 0; i < count; i++) sum += rows[7].id; + return sum; +} + +const edgeRows: any = JSON.parse( + '[{"id":0},{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},' + + '{"id":-0},{"id":1e308},{"id":-1e308},{"id":2.5}]', +); +// `-0 + -0` is `-0`; any `+0` in the chain makes it `+0`. +const negZero = repeatFrom(edgeRows, 3, -0); +console.log("repeat-negative-zero:", negZero, Object.is(negZero, -0)); +const posZero = repeatFrom(edgeRows, 3, 0); +console.log("repeat-positive-zero:", posZero, Object.is(posZero, -0)); +// A non-Number entry value takes the slow clone and keeps concatenation. +const sevenRows: any = JSON.parse(buildRecords(10, "p")); +console.log("repeat-string-start:", repeatFrom(sevenRows, 3, "s")); + +// Overflow to Infinity, then Infinity + -Infinity = NaN, in source order. +function sequentialFrom(rows: any, count: number, n: number, start: number): number { + let sum = start; + for (let i = 0; i < count; i++) { + const index = i % n; + sum += rows[index].id; + } + return sum; +} +const overflowRows: any = JSON.parse('[{"id":1e308},{"id":1e308},{"id":-1e308}]'); +console.log("overflow-to-infinity:", sequentialFrom(overflowRows, 2, 3, 0)); +console.log("overflow-then-back:", sequentialFrom(overflowRows, 3, 3, 0)); +const infRows: any = JSON.parse('[{"id":1e999},{"id":-1e999}]'); +console.log("infinity-minus-infinity:", sequentialFrom(infRows, 2, 2, 0)); +console.log("nan-start:", sequentialFrom(overflowRows, 3, 3, NaN)); + +// A side exit on the FIRST iteration with a non-zero accumulator: the +// trampoline must publish the entry value, not the untouched pre-loop slot. +const stringAtSeven: any = JSON.parse( + '[{"id":0},{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":"x"}]', +); +function repeatAfterPrefix(rows: any, count: number): any { + let sum: any = 0; + for (let i = 0; i < 4; i++) sum += rows[i].id; + for (let i = 0; i < count; i++) sum += rows[7].id; + return sum; +} +console.log("repeat-side-exit-after-prefix:", repeatAfterPrefix(stringAtSeven, 3)); + +// A side exit MID-loop with the counter-indexed form: the published +// accumulator is the numeric prefix, and the slow clone runs exactly the +// remaining iterations — both visible in the concatenated result. +const lateString: any = JSON.parse( + '[{"id":1.5},{"id":2},{"id":3},{"id":4},{"id":"five"},{"id":6},{"id":7}]', +); +console.log("scan-side-exit-mid-loop:", scanSum(lateString, lateString.length)); +console.log("sequential-side-exit-mid-loop:", sequentialFrom(lateString, 9, 7, 10)); + +// The `repeat` counter against bounds the i32 materialization must refuse: +// every one takes the slow clone and keeps JavaScript's trip count. +console.log("repeat-fractional-count:", repeatFrom(sevenRows, 2.5, 0)); +console.log("repeat-nan-count:", repeatFrom(sevenRows, NaN, 0)); +console.log("repeat-negative-count:", repeatFrom(sevenRows, -3, 0)); +console.log("repeat-zero-count:", repeatFrom(sevenRows, 0, 1)); +console.log("repeat-string-count:", repeatFrom(sevenRows, "3" as any, 0)); +console.log("repeat-integral-count:", repeatFrom(sevenRows, 3, 0.5)); + +// The `random` recurrence with a side exit well past the first iteration: the +// published counter decides how many iterations the slow clone still runs. +function randomCounted(rows: any, count: number, n: number): string { + let sum: any = 0; + let cursor = 0; + let seen = 0; + for (let i = 0; i < count; i++) { + cursor = (cursor * 17 + 7) % n; + sum += rows[cursor].id; + } + for (let i = 0; i < count; i++) seen++; + return String(sum) + "|" + String(cursor) + "|" + String(seen); +} +// `(17c + 7) % 13` walks 7 -> 9 -> 4 -> 10 -> 8 -> 0 from 0, so index 8 — the +// string one — is read on the FIFTH iteration and every sixth after it. +const randomLate: any = JSON.parse( + '[{"id":0},{"id":1},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},' + + '{"id":7},{"id":"eight"},{"id":9},{"id":10},{"id":11},{"id":12}]', +); +console.log("random-late-side-exit:", randomCounted(randomLate, 40, 13)); From 7928189c7e2a5c59d64cc8b4a1872bc924022ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:42:34 +0200 Subject: [PATCH 4/7] style(codegen): rustfmt module order --- crates/perry-codegen/src/stmt/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/perry-codegen/src/stmt/mod.rs b/crates/perry-codegen/src/stmt/mod.rs index 18e645070f..a6ca584e6a 100644 --- a/crates/perry-codegen/src/stmt/mod.rs +++ b/crates/perry-codegen/src/stmt/mod.rs @@ -22,9 +22,9 @@ mod class_field_loop_tests; mod counter_range; mod element_shape_carried; mod element_shape_loop; -mod element_shape_native; #[cfg(test)] mod element_shape_loop_tests; +mod element_shape_native; mod if_stmt; mod let_buffer_views; mod let_object_facts; From 2e907166ab124620caa395a2ebc4dea332dbeac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 07:43:44 +0200 Subject: [PATCH 5/7] changelog: element-shape clone native accumulator and counter (#10255) --- .../10255-element-shape-native-domains.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 changelog.d/10255-element-shape-native-domains.md diff --git a/changelog.d/10255-element-shape-native-domains.md b/changelog.d/10255-element-shape-native-domains.md new file mode 100644 index 0000000000..6f9699421c --- /dev/null +++ b/changelog.d/10255-element-shape-native-domains.md @@ -0,0 +1,105 @@ +Keep the element-shape loop clone's accumulator in the double domain and its +counter in the i32 domain (#7480 → #10171 → #10185). The JSON access screen's +four missing cells (`repeat` ×3, 16k `fields`) now beat Node and Bun at the +official settings, and every clone form got cheaper. + +**The defect.** The fast clone was already what executed, but the `repeat` loop +(`sum += rows[7].id`) took 16 instructions at ~13 CPU cycles per iteration: two +loop-carried chains crossed the integer/float register boundary every +iteration. + +* An `any`-typed `sum` lives in a precise GC root slot, and every root reload + passes through the RS4GC launder (`function/precise_roots.rs`, + `ROOT_RELOAD_LAUNDER`) that LLVM cannot see through, so mem2reg promoted the + slot as NaN-box `i64` bits: `x25 → fmov → fadd → fmov → x25`. +* A counter the clone never indexes with (the constant-index and carried forms) + had no i32 slot, so it stayed a double (`fadd d8, #1.0`) compared against + `count` — itself a root slot, reloaded through the launder and moved to an FP + register every iteration although the preheader had already materialized it + as an i32. + +**The change** (`stmt/element_shape_native.rs`). For the fast clone's lowering +the accumulator is redirected into a promotable `alloca double` (the +`numeric_accumulator_f64_slots` redirect the packed clones already use), seeded +with the value the deref block tag-tested as a Number; the counter gets an i32 +slot (its Let-site parallel slot, or a clone-private one seeded with the literal +start) that the `Update` lowering advances alone +(`deferred_integer_update_accumulators`), so the precomputed i32 trip count +turns the condition into `icmp slt i32`. Residual checks branch to a +`element_shape.loop.side_exit` trampoline that publishes both scalars to their +real slots and then enters the slow clone; the fall-through exit publishes them +in `element_shape.loop.fast.write_back`. A canonical-i32 counter (the indexing +forms) has nothing to defer and is left alone. + +Soundness is unchanged by construction: the redirected store sits exactly where +the root-slot store did, which #10185's fold / carried-commit ordering already +put after every side exit of the iteration, so the trampoline publishes the +iteration's entry accumulator and its own index — the state the slow clone +re-runs it from. The carried binding keeps its own end-of-iteration commit and +is never published by the trampoline. The seed, trampoline and write-back are +plain loads/stores plus one `sitofp`, created inside the call-free scan's block +range. The accumulator was already consumed as a raw double inside this clone +(`numeric_accumulator`), so the IEEE `fadd`s are the same operations on the same +operands in the same order (`-0`, NaN, overflow to Infinity bit-identical). The +i32 counter only ever runs against a trip count that is a literal, `arr.length` +or `materialize_loop_i32`'s integral `0..=i32::MAX`; fractional, NaN, negative +and out-of-range bounds still route to the slow clone. + +**The new `repeat` loop** (arm64, `run$spec_b_b`, whole body): + +``` +a90 cbz w9, side_exit ; residual check (header loads hoisted) +a94 ldr d2, [x10, w0, sxtw #3] ; field load +a98 fmov x12, d2 +a9c cmp x12, x11 ; Number tag test +aa0 b.gt side_exit +aa4 fadd d1, d1, d2 ; sum stays in d1 +aa8 add w8, w8, #1 ; i is an i32 +aac cmp w19, w8 ; against the materialized i32 count +ab0 b.ne a90 +``` + +**Per iteration** (`/usr/bin/time -l`, 50M iterations, 16k fixture): + +| mode | instructions main → PR | cycles main → PR | +|---|---:|---:| +| repeat | 16.0 → 9.0 | 13.0 → 3.0 | +| sequential | 25.0 → 22.0 | 13.0 → 3.4 | +| random | 29.0 → 27.0 | 15.6 → 15.6 | +| fields | 51.0 → 47.0 | 19.0 → 9.0 | + +**Access screen, official settings** (1M iterations, warmup 0; bench mini, best +of 9 interleaved rounds, ns/iteration): `repeat` 4.06 → 0.94 (Node 2.89–3.07, +Bun 4.47–4.60); `sequential` 4.06–4.16 → 1.07/1.63/3.31; `random` 4.86–5.45 → +4.80–5.39; `fields` 5.94–6.03 → 2.81/2.82/3.39 (16k Node 5.57). All 12 cells at +or better than the better of Node and Bun (worst ratio 0.70, 16k `random`). + +**Warmed** (50M iterations, 1M warmup, best of 5): 3 of 12 cells win (16k +`sequential`, 1m and 20m `fields`); 9 still miss, each for a structural reason +this change does not address: + +* `repeat` 0.94 vs 0.38/0.52/0.31 — one IEEE add per iteration is 3 cycles of + latency on this core; the JITs' 1.0–1.6 cycles mean an int32 speculated + accumulator, which JS double semantics for `sum` do not license here. +* `fields` 16k 2.81 vs Node 2.77 — three serial IEEE adds = 9 cycles (measured + 9.02), the double-domain floor for the source-order fold. +* `random` 4.86–5.39 vs 4.47–5.34 — the loop-carried chain is the recurrence's + division (15.6 cycles, unchanged); the i64 `srem` #10185 needs for exactness is + ~5 % slower than the int32 division a JIT emits (same-host C microbenchmark: + 4.69 vs 4.46 ns), plus random-access cache misses on 1m/20m. +* `sequential` 1m/20m 1.65/3.38 vs 1.32/2.02 — identical 22 instructions but + 3.4 → 5.2 → 10.9 cycles from 16k to 20m: memory-bound record access. + +**JSON matrix** (50 rows, 5 interleaved rounds, PR vs main): 49 rows within +±2 % CPU (the one element-shape consumer, `scan`, 0.5–1.1 % faster), peak RSS +identical. `escaped_1m:stringify` read +10 % (160.8 → 177.3 ms); its hot function +`json::stringify_flat::emit_piece` is identical runtime code shifted 128 bytes +by the smaller worker module, and appending unused functions to main's own +worker moves the same row to 149.0–149.4 ms — layout, not this change. + +Tests: five IR-census tests in `stmt/element_shape_native_tests.rs` (each fails +with the redirects disabled); the #10185 carried-commit and side-exit helpers +now count the trampoline spelling of an exit. `test_gap_json_record_loop_clone.ts` +gains `-0`/Infinity/NaN accumulation, first-iteration and mid-loop side exits +with a non-zero accumulator, fractional/NaN/negative/string trip counts on the +`repeat` form and a late `random` side exit; byte-identical to Node 26.5.1. From 5dc5798c3760df5ad529f5fcf2111a415391256d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 08:06:01 +0200 Subject: [PATCH 6/7] changelog: state the matrix distribution exactly --- changelog.d/10255-element-shape-native-domains.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/changelog.d/10255-element-shape-native-domains.md b/changelog.d/10255-element-shape-native-domains.md index 6f9699421c..b20535e3c4 100644 --- a/changelog.d/10255-element-shape-native-domains.md +++ b/changelog.d/10255-element-shape-native-domains.md @@ -90,9 +90,11 @@ this change does not address: * `sequential` 1m/20m 1.65/3.38 vs 1.32/2.02 — identical 22 instructions but 3.4 → 5.2 → 10.9 cycles from 16k to 20m: memory-bound record access. -**JSON matrix** (50 rows, 5 interleaved rounds, PR vs main): 49 rows within -±2 % CPU (the one element-shape consumer, `scan`, 0.5–1.1 % faster), peak RSS -identical. `escaped_1m:stringify` read +10 % (160.8 → 177.3 ms); its hot function +**JSON matrix** (50 rows, 5 interleaved rounds, best-of, PR vs main): 48 rows +within ±2 % CPU (the one element-shape consumer, `scan`, 0.5–1.1 % faster), no +row's peak RSS higher. Two runtime-only rows moved further, one each way: +`long_string_1m:stringify` −4.4 % and `escaped_1m:stringify` +10.3 % (160.8 → +177.3 ms). Neither runs a clone; the latter's hot function `json::stringify_flat::emit_piece` is identical runtime code shifted 128 bytes by the smaller worker module, and appending unused functions to main's own worker moves the same row to 149.0–149.4 ms — layout, not this change. From 8ba5e164ba98b359a78bb754cf433f3a725577d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 08:06:21 +0200 Subject: [PATCH 7/7] changelog: separate the random gap's division share from its locality share --- changelog.d/10255-element-shape-native-domains.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/changelog.d/10255-element-shape-native-domains.md b/changelog.d/10255-element-shape-native-domains.md index b20535e3c4..303930660e 100644 --- a/changelog.d/10255-element-shape-native-domains.md +++ b/changelog.d/10255-element-shape-native-domains.md @@ -83,12 +83,16 @@ this change does not address: accumulator, which JS double semantics for `sum` do not license here. * `fields` 16k 2.81 vs Node 2.77 — three serial IEEE adds = 9 cycles (measured 9.02), the double-domain floor for the source-order fold. -* `random` 4.86–5.39 vs 4.47–5.34 — the loop-carried chain is the recurrence's - division (15.6 cycles, unchanged); the i64 `srem` #10185 needs for exactness is - ~5 % slower than the int32 division a JIT emits (same-host C microbenchmark: - 4.69 vs 4.46 ns), plus random-access cache misses on 1m/20m. +* `random` 4.86/5.09/5.39 vs 4.63/4.47/4.64 — the loop-carried chain is the + recurrence's division (15.6 cycles, unchanged by this change); the i64 `srem` + #10185 needs for exactness is ~5 % slower than an int32 division (same-host C + microbenchmark: 4.69 vs 4.46 ns), which is the whole 16k gap. The larger 1m/20m + gap is not in the loop code: Perry's 27 instructions are identical across + sizes while its cycles rise 15.6 → 16.3 → 17.3, and Bun gets FASTER than its + own 16k number — a record-memory locality difference, not investigated here. * `sequential` 1m/20m 1.65/3.38 vs 1.32/2.02 — identical 22 instructions but - 3.4 → 5.2 → 10.9 cycles from 16k to 20m: memory-bound record access. + 3.4 → 5.2 → 10.9 cycles from 16k to 20m: memory-bound record access, the same + locality difference. **JSON matrix** (50 rows, 5 interleaved rounds, best-of, PR vs main): 48 rows within ±2 % CPU (the one element-shape consumer, `scan`, 0.5–1.1 % faster), no