From edcdb2eea529d2a3e4d5a974aa5ba45798066778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 18 Sep 2026 10:34:51 +0000 Subject: [PATCH 1/2] perf(gc): gate the residual prototype registry on an owner header bit (#10362) The registry holding an explicit [[Prototype]] for a non-meta-capable owner was gated only by OBJECT_PROTOTYPES_NONEMPTY, a process-global latch. One re-prototyped object anywhere armed it for the rest of the run, after which every traced owner-capable cell paid a lock plus a SipHash probe to ask a question that is false for almost all of them. A latch is a cliff: it turns the fast path off for every cell at once, invisibly to any benchmark that does not contain the trigger. Bit 6 of _reserved is OBJ_FLAG_NULL_PROTO, which has exactly one setter (returning *mut ObjectHeader) and seven readers, every one provably unreachable with a non-GC_TYPE_OBJECT cell: three by an explicit obj_type check, three by a converter that returns None first, one by a preceding conjunct in the same && chain. The registry excludes GC_TYPE_OBJECT by construction, so the bit is free across the registry's whole population, not only for arrays -- which is why both existing witnesses exercise it, one of them a lazy array that an array-scoped bit would have missed. GC_RESIDUAL_PROTO_OWNER is set at the single funnel, under the registry lock and before the insert: the proof is published before the fact it guards. It is never cleared, and that is sound. Entries outlive owners only when the owner is dead; the prune touches only dead owners; both rekey paths keep the entry while _reserved rides the move (#10381's contract, enforced by assert_relocation_copied_the_header). One writer under one lock writes both, so the dangerous direction -- entry present, bit absent -- has no producer. A GC_TYPE_OBJECT owner that reaches the registry anyway keeps the latch-only gate, since bit 6 means something else there. The latch stays as the first test -- one byte load, false for any process that never re-prototyped a non-object -- and the bit is the second, which is what stops an ARMED process paying per traced cell. Sabotage: with the setter made a process-wide no-op, both existing witnesses in gc/tests/residual_prototype_relocation.rs fail at their real verdicts, the registry entry no longer following the lazy header nor the array owner. The bit is load-bearing, not decorative. Measured on main 9df5075fb, exact instruction counts: the fixture that arms the latch -0.408%, and three that do not are flat (+0.015%, -0.060%, -0.021%). Attributed: -94.3M RandomState::hash_one, -58.2M SipHash write, -36.9M run_copied_minor_attempt, -30.0M transfer_residual_prototype. pointer_slots_read is identical between arms: the collector does bit-identical work. --- .../perry-runtime/src/gc/layout/transfer.rs | 9 +- .../perry-runtime/src/gc/layout_slot_visit.rs | 6 + .../gc/tests/residual_prototype_relocation.rs | 74 +++++++++ crates/perry-runtime/src/gc/types.rs | 38 +++++ .../src/object/prototype_chain.rs | 146 +++++++++++++++++- 5 files changed, 268 insertions(+), 5 deletions(-) diff --git a/crates/perry-runtime/src/gc/layout/transfer.rs b/crates/perry-runtime/src/gc/layout/transfer.rs index e8eb7bbd98..c4fc849f45 100644 --- a/crates/perry-runtime/src/gc/layout/transfer.rs +++ b/crates/perry-runtime/src/gc/layout/transfer.rs @@ -95,10 +95,17 @@ pub(crate) unsafe fn layout_transfer(old_user: *mut u8, new_user: *mut u8) { // layout kinds (module docs). The latch first: it is one byte load, false // for any process that never re-prototyped a non-object, and the move is // out of line. + let relocating_header = header_from_user_ptr(old_user as *const u8); if crate::object::prototype_chain::object_static_prototypes_maybe_nonempty() && crate::object::prototype_chain::residual_prototype_owner_type( - (*header_from_user_ptr(old_user as *const u8)).obj_type, + (*relocating_header).obj_type, ) + // #10362: the per-OWNER half, same bit and same proof as the trace + // path's. `_reserved` rides every relocation by construction — all four + // callers copy it before calling here and + // `assert_relocation_copied_the_header` enforces that — so the bit is + // already correct at this point and needs no transfer of its own. + && crate::object::prototype_chain::residual_entry_possible_for(relocating_header) { transfer_residual_prototype(old_user as usize, new_user as usize); } diff --git a/crates/perry-runtime/src/gc/layout_slot_visit.rs b/crates/perry-runtime/src/gc/layout_slot_visit.rs index d76398e700..20cad9ad67 100644 --- a/crates/perry-runtime/src/gc/layout_slot_visit.rs +++ b/crates/perry-runtime/src/gc/layout_slot_visit.rs @@ -167,6 +167,12 @@ pub(super) unsafe fn visit_gc_rewrite_slot_descriptors( // arms, so no arm's early return can skip it. if crate::object::prototype_chain::object_static_prototypes_maybe_nonempty() && crate::object::prototype_chain::residual_prototype_owner_type(obj_type) + // #10362: the per-OWNER half. The latch above is exact for a process + // that never re-prototyped a non-object and useless for one that has — + // it is what made a single `Object.setPrototypeOf(anArray, p)` charge + // every traced cell of every owner-capable kind a global mutex and a + // SipHash probe. This asks the owner's own header instead. + && crate::object::prototype_chain::residual_entry_possible_for(header) { crate::object::prototype_chain::visit_object_static_prototype_slot_mut( user_ptr as usize, diff --git a/crates/perry-runtime/src/gc/tests/residual_prototype_relocation.rs b/crates/perry-runtime/src/gc/tests/residual_prototype_relocation.rs index 2340ea7f0a..12cb6ce779 100644 --- a/crates/perry-runtime/src/gc/tests/residual_prototype_relocation.rs +++ b/crates/perry-runtime/src/gc/tests/residual_prototype_relocation.rs @@ -175,6 +175,80 @@ fn test_lazy_array_explicit_prototype_survives_a_copying_minor() { ); } +/// #10362 SABOTAGE: with `GC_RESIDUAL_PROTO_OWNER` never set, the per-owner +/// gates answer "no entry here" and the collector skips the prototype edge. +/// +/// This is the test that proves the bit is load-bearing rather than +/// documentation. Its subject is the same one the two tests around it assert on +/// — the entry must follow the owner and the recorded address must be rewritten +/// — so a green run here with the bit suppressed would mean the gates it feeds +/// are not gating anything, and A' should be withdrawn. +/// +/// The membership assertion (`debug_assert_residual_owner_bit`) stands itself +/// down while the sabotage is armed, so this test fails at the real verdict +/// rather than at an assertion the sabotage itself provoked. +#[test] +fn test_suppressing_the_residual_owner_bit_loses_the_prototype() { + let _serialized = crate::array::test_serialize(); + let _feedback = crate::typed_feedback::typed_feedback_test_lock(); + let _latch = ArrayPrototypeLatchRestore::capture(); + let _guard = CopyingNurseryTestGuard::new(1); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + + let owner = crate::array::js_array_alloc(4) as usize; + let obj_type = obj_type_at(owner); + assert!( + crate::arena::pointer_in_nursery(owner) && crate::gc::gc_type_is_movable(obj_type), + "premise: a nursery owner of a movable kind" + ); + assert_ne!( + obj_type, GC_TYPE_OBJECT, + "premise: the subject must be a kind the bit actually gates" + ); + js_shadow_slot_set(0, ptr_bits(owner)); + + let sabotage = crate::object::prototype_chain::residual_proto_bit_sabotage::Guard::arm(); + let proto = marked_prototype(); + crate::object::js_object_set_prototype_of( + f64::from_bits(ptr_bits(owner)), + f64::from_bits(ptr_bits(proto)), + ); + let owner = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_eq!( + crate::object::prototype_chain::object_static_prototype(owner), + Some(ptr_bits(proto)), + "premise: the entry is recorded even with the bit suppressed — the \ + sabotage removes the PROOF, not the entry" + ); + unsafe { + let header = + (owner as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + assert_eq!( + (*header)._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER, + 0, + "premise: the sabotage really did suppress the bit" + ); + } + + let trace = collect_minor_trace(GcTriggerKind::Direct); + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + let owner_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(owner_after, owner, "premise: the owner must actually move"); + + let recorded = crate::object::prototype_chain::object_static_prototype(owner_after); + forget_owners(&[owner, owner_after]); + js_shadow_slot_set(0, 0); + drop(sabotage); + + assert!( + recorded.is_none(), + "the bit is not load-bearing: with `GC_RESIDUAL_PROTO_OWNER` suppressed the \ + entry still followed its owner, so the per-owner gates in \ + `gc/layout_slot_visit.rs` and `gc/layout/transfer.rs` are not gating \ + anything. A' is documentation — withdraw it." + ); +} + /// Every movable receiver kind that keeps its prototype in the residual /// registry, with the prototype held by NOTHING but that entry. One copying /// minor has to rekey the entry, retain the prototype through it, and rewrite diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index 835cb92074..2bbe5eaa2e 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -1294,6 +1294,44 @@ pub const OBJ_FLAG_TYPED_ARRAY_PROTO: u16 = 0x100; /// `JSValue` slots. This is only meaningful for `GC_TYPE_ARRAY`; object /// flags share the same `_reserved` word but never inspect this bit. pub(crate) const GC_ARRAY_RAW_F64_LAYOUT: u16 = 0x80; +/// #10362: this cell owns an entry in the residual static-prototype registry +/// (`object::prototype_chain`) — i.e. an `Object.setPrototypeOf` whose receiver +/// `meta_capable_object` turned away, so the prototype could not go in a meta +/// record and went into the address-keyed table instead. +/// +/// The registry's readers used to ask only the process-global +/// `OBJECT_PROTOTYPES_NONEMPTY` latch, which is exact for a process that has +/// never re-prototyped a non-object and useless for one that has: a single +/// `Object.setPrototypeOf(anArray, p)` made EVERY traced cell of every +/// owner-capable kind take the registry's global mutex and a SipHash probe, and +/// every relocation of one call the rekey hook. Measured on a 200k-array +/// fixture: 400,000 armed hook calls from three lines of setup, against 155 +/// unarmed. This bit answers the same question per OWNER, so an armed process +/// pays only for the cells that actually have an entry. +/// +/// Bit 6, shared with `OBJ_FLAG_NULL_PROTO` exactly as bits 7..12 are already +/// shared between the OBJECT and ARRAY namespaces, and disjoint from it by +/// `obj_type`: `OBJ_FLAG_NULL_PROTO` has one setter, +/// `js_object_alloc_null_proto`, which returns `*mut ObjectHeader`, and every +/// reader of it is behind an `obj_type == GC_TYPE_OBJECT` guard (audited: +/// `field_get_set/accessors.rs`, `field_get_set/for_in_stable.rs`, +/// `field_set_by_name.rs`, `field_set_by_name/tail.rs` via `object_is_regular`, +/// `builtins/formatting.rs` via both callers' `gc_type` dispatch, +/// `builtins/formatting/prototype_equality.rs` via `heap_object_addr`, +/// `native_call_method/object_proto.rs` via `object_ptr_from_value`). +/// So this bit is **only meaningful for `obj_type != GC_TYPE_OBJECT`**, and a +/// `GC_TYPE_OBJECT` owner that reaches the registry anyway keeps the +/// latch-only gate (see `residual_entry_possible_for`). +/// +/// Set-only, like `GC_ARRAY_NAMED_PROPS` and for the same reason: an entry is +/// never deleted while its owner lives (a second `setPrototypeOf` overwrites +/// the same key; the prune only removes DEAD owners, whose header is gone; the +/// two rekey paths remove-then-insert and the entry survives). So the error is +/// on the safe side by construction — a stale-set bit costs one probe that the +/// map answers `None` to, while the dangerous direction (entry present, bit +/// absent) has no code path that can produce it, because the only writer of the +/// entry is also the only writer of the bit, under one lock. +pub(crate) const GC_RESIDUAL_PROTO_OWNER: u16 = 0x40; /// Array was synthesized for a function's `arguments` binding. This is only /// meaningful for `GC_TYPE_ARRAY`; it lets `util.types.isArgumentsObject` /// distinguish Perry's internal `arguments` arrays from user rest arrays. diff --git a/crates/perry-runtime/src/object/prototype_chain.rs b/crates/perry-runtime/src/object/prototype_chain.rs index 3376c3a07b..ff8f7a20eb 100644 --- a/crates/perry-runtime/src/object/prototype_chain.rs +++ b/crates/perry-runtime/src/object/prototype_chain.rs @@ -160,6 +160,130 @@ pub(crate) fn any_user_prototype_override() -> bool { USER_PROTO_OVERRIDE_EVER.load(Ordering::Acquire) } +/// #10362: mark `obj_ptr`'s own header as an owner in the residual registry. +/// +/// Called under the registry lock and BEFORE the insert, the same discipline +/// `OBJECT_PROTOTYPES_NONEMPTY` uses one line below: the proof is published +/// before the fact it guards, so a reader that can observe the entry already +/// observes the bit. +/// +/// Silently does nothing for a `GC_TYPE_OBJECT` owner. Such an owner normally +/// never reaches the registry at all (`meta_capable_object` takes it), but it +/// can when that function turns it away for a non-type reason — and bit 6 means +/// `OBJ_FLAG_NULL_PROTO` there, so it must not be reused. Those owners keep the +/// latch-only gate, which is what every owner had before this change. +unsafe fn set_residual_proto_owner_bit(obj_ptr: usize) { + #[cfg(test)] + if residual_proto_bit_sabotage::suppressed() { + return; + } + let Some(header) = crate::value::addr_class::try_read_gc_header(obj_ptr) else { + return; + }; + if header.obj_type == crate::gc::GC_TYPE_OBJECT { + return; + } + let header = (obj_ptr as *mut u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; + (*header)._reserved |= crate::gc::GC_RESIDUAL_PROTO_OWNER; +} + +/// Can the cell at `header` own a residual-prototype entry, judged from its own +/// header rather than from the process-global latch? +/// +/// This is the per-owner half of the registry's gate. Callers keep asking +/// [`object_static_prototypes_maybe_nonempty`] FIRST — it is one byte load and +/// false for any process that never re-prototyped a non-object — and ask this +/// second, which is what stops an ARMED process paying per traced cell. +/// +/// Conservative for `GC_TYPE_OBJECT`: see `set_residual_proto_owner_bit`. +/// +/// # Safety +/// +/// `header` is a readable `GcHeader` of a live allocation. +#[inline] +pub(crate) unsafe fn residual_entry_possible_for(header: *const crate::gc::GcHeader) -> bool { + if (*header).obj_type == crate::gc::GC_TYPE_OBJECT { + return true; + } + (*header)._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER != 0 +} + +/// The invariant the collector's gates rest on: a LIVE non-object owner that +/// has an entry in the registry carries the bit. +/// +/// Asserted in test and debug builds — including `cargo test --release`, how the +/// GC suites run — for the same reason +/// `gc::layout::transfer::assert_relocation_copied_the_header` is: a test proves +/// today's code, an assertion proves tomorrow's. A future path that inserts an +/// entry without the bit would make every collector gate skip that owner's +/// prototype edge, which is #10493's bug exactly — correct before a collection, +/// wrong after, exit code 0 and no warning. +/// +/// Only this direction is an invariant. The reverse (bit set implies an entry) +/// is deliberately NOT asserted: the bit is set-only, and the two rekey paths +/// remove-then-insert with the lock released in between, so a bit without an +/// entry is a legal transient and a benign steady state. +#[inline] +pub(crate) unsafe fn debug_assert_residual_owner_bit(obj_ptr: usize) { + #[cfg(any(test, debug_assertions))] + { + #[cfg(test)] + if residual_proto_bit_sabotage::suppressed() { + return; + } + if let Some(header) = crate::value::addr_class::try_read_gc_header(obj_ptr) { + if header.obj_type != crate::gc::GC_TYPE_OBJECT { + assert!( + header._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER != 0, + "residual prototype registry: owner {obj_ptr:#x} (obj_type {}) has an \ + entry but not `GC_RESIDUAL_PROTO_OWNER`, so every collector gate will \ + skip its prototype edge — the prototype is neither retained nor \ + rewritten (#10493's failure mode)", + header.obj_type + ); + } + } + } + #[cfg(not(any(test, debug_assertions)))] + { + let _ = obj_ptr; + } +} + +/// Test-only sabotage for [`set_residual_proto_owner_bit`]: the bit is never +/// set, so every per-owner gate falls back to "no entry here" and the collector +/// skips the prototype edge. +/// +/// Both tests in `gc/tests/residual_prototype_relocation.rs` MUST fail while +/// this is armed. If they pass, the bit is not load-bearing and is +/// documentation — the failure mode CLAUDE.md calls "a gate that cannot fail". +#[cfg(test)] +pub(crate) mod residual_proto_bit_sabotage { + use std::cell::Cell; + + thread_local! { + static SUPPRESSED: Cell = const { Cell::new(false) }; + } + + pub(crate) fn suppressed() -> bool { + SUPPRESSED.with(Cell::get) + } + + pub(crate) struct Guard(bool); + + impl Guard { + pub(crate) fn arm() -> Self { + Self(SUPPRESSED.with(|s| s.replace(true))) + } + } + + impl Drop for Guard { + fn drop(&mut self) { + SUPPRESSED.with(|s| s.set(self.0)); + } + } +} + fn get_object_prototypes() -> &'static Mutex> { OBJECT_PROTOTYPES.get_or_init(|| Mutex::new(HashMap::new())) } @@ -325,6 +449,9 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, link_kind: // The publish property is unchanged: a reader that sees `true` takes // the lock and therefore sees whatever the writer committed. OBJECT_PROTOTYPES_NONEMPTY.store(true, Ordering::Release); + // #10362: the per-OWNER half of the same proof, published under the + // same lock and before the same insert, for the same reason. + unsafe { set_residual_proto_owner_bit(obj_ptr) }; let slot = map.entry(obj_ptr).or_insert(0); *slot = proto_bits; slot_addr = slot as *mut u64 as usize; @@ -360,10 +487,17 @@ pub fn object_static_prototype(obj_ptr: usize) -> Option { if !OBJECT_PROTOTYPES_NONEMPTY.load(Ordering::Acquire) { return None; } - get_object_prototypes() + let recorded = get_object_prototypes() .lock() .ok() - .and_then(|map| map.get(&obj_ptr).copied()) + .and_then(|map| map.get(&obj_ptr).copied()); + // #10362: the invariant every collector gate rests on, checked on the read + // paths that are NOT gated by the bit — asserting it inside a bit-gated + // path would be vacuous. + if recorded.is_some() { + unsafe { debug_assert_residual_owner_bit(obj_ptr) }; + } + recorded } /// Look up the residual prototype registry for a caller that has already @@ -383,10 +517,14 @@ pub(crate) fn object_static_prototype_known_non_meta(obj_ptr: usize) -> Option Date: Fri, 18 Sep 2026 12:36:03 +0200 Subject: [PATCH 2/2] changelog: fragment for #10611 --- changelog.d/10611-residual-proto-owner-bit.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog.d/10611-residual-proto-owner-bit.md diff --git a/changelog.d/10611-residual-proto-owner-bit.md b/changelog.d/10611-residual-proto-owner-bit.md new file mode 100644 index 0000000000..289bf29f23 --- /dev/null +++ b/changelog.d/10611-residual-proto-owner-bit.md @@ -0,0 +1,9 @@ +### Performance + +- **The residual prototype registry is gated on an owner header bit instead of a process-global latch (#10362).** An explicit `[[Prototype]]` on a non-meta-capable owner lives in an address-keyed registry (#9304) whose relocation obligations were gated only by `OBJECT_PROTOTYPES_NONEMPTY`. One re-prototyped object anywhere armed that latch for the rest of the run, and every traced owner-capable cell then paid a lock plus a SipHash probe to ask a question false for almost all of them — a cliff, not a slope, invisible to any benchmark without the trigger. Measured: routine code leaves it alone (`class X extends Error`, live Array/Map/Set/Error subclass instances: 372–405 relocation-hook calls), while one `Object.setPrototypeOf` on an array takes it to 400,000. + + `GC_RESIDUAL_PROTO_OWNER` reuses bit 6 of `_reserved`, which is `OBJ_FLAG_NULL_PROTO` for `GC_TYPE_OBJECT` and free for every other kind — audited to all seven readers, each provably unreachable with a non-object cell (three by an explicit `obj_type` check, three by a converter returning `None` first, one by a preceding conjunct). Since the registry excludes `GC_TYPE_OBJECT` by construction the bit covers the registry's whole population, which is why both existing witnesses exercise it — one of them a lazy array an array-scoped bit would have missed. Set-only at the single funnel, under the registry lock and before the insert, so the proof is published before the fact it guards; never cleared, because the dangerous direction (entry present, bit absent) has no producer when one writer under one lock writes both. The latch stays as a one-load first test for processes that never re-prototype anything. + + Sabotage: with the setter made a no-op, both existing relocation witnesses fail at their real verdicts. The invariant is additionally asserted under `cfg(any(test, debug_assertions))`, which does run under `cargo test --release`. + + Measured exactly: the fixture that arms the latch −0.408%, three that do not are flat (+0.015%, −0.060%, −0.021%), attributed to −94.3M `RandomState::hash_one`, −58.2M SipHash `write`, −36.9M `run_copied_minor_attempt` and −30.0M `transfer_residual_prototype`. `pointer_slots_read` is identical between arms: the collector does bit-identical work.