perf(gc): gate the residual prototype registry on an owner header bit (#10362)

changelog.d/10611-residual-proto-owner-bit.md

@@ -0,0 +1,9 @@
+### Performance
+
+- **The residual prototype registry is gated on an owner header bit instead of a process-global latch (#10362).** An explicit `[[Prototype]]` on a non-meta-capable owner lives in an address-keyed registry (#9304) whose relocation obligations were gated only by `OBJECT_PROTOTYPES_NONEMPTY`. One re-prototyped object anywhere armed that latch for the rest of the run, and every traced owner-capable cell then paid a lock plus a SipHash probe to ask a question false for almost all of them — a cliff, not a slope, invisible to any benchmark without the trigger. Measured: routine code leaves it alone (`class X extends Error`, live Array/Map/Set/Error subclass instances: 372–405 relocation-hook calls), while one `Object.setPrototypeOf` on an array takes it to 400,000.
+
+  `GC_RESIDUAL_PROTO_OWNER` reuses bit 6 of `_reserved`, which is `OBJ_FLAG_NULL_PROTO` for `GC_TYPE_OBJECT` and free for every other kind — audited to all seven readers, each provably unreachable with a non-object cell (three by an explicit `obj_type` check, three by a converter returning `None` first, one by a preceding conjunct). Since the registry excludes `GC_TYPE_OBJECT` by construction the bit covers the registry's whole population, which is why both existing witnesses exercise it — one of them a lazy array an array-scoped bit would have missed. Set-only at the single funnel, under the registry lock and before the insert, so the proof is published before the fact it guards; never cleared, because the dangerous direction (entry present, bit absent) has no producer when one writer under one lock writes both. The latch stays as a one-load first test for processes that never re-prototype anything.
+
+  Sabotage: with the setter made a no-op, both existing relocation witnesses fail at their real verdicts. The invariant is additionally asserted under `cfg(any(test, debug_assertions))`, which does run under `cargo test --release`.
+
+  Measured exactly: the fixture that arms the latch −0.408%, three that do not are flat (+0.015%, −0.060%, −0.021%), attributed to −94.3M `RandomState::hash_one`, −58.2M SipHash `write`, −36.9M `run_copied_minor_attempt` and −30.0M `transfer_residual_prototype`. `pointer_slots_read` is identical between arms: the collector does bit-identical work.

crates/perry-runtime/src/gc/layout/transfer.rs

@@ -95,10 +95,17 @@ pub(crate) unsafe fn layout_transfer(old_user: *mut u8, new_user: *mut u8) {
     // layout kinds (module docs). The latch first: it is one byte load, false
     // for any process that never re-prototyped a non-object, and the move is
     // out of line.
+    let relocating_header = header_from_user_ptr(old_user as *const u8);
     if crate::object::prototype_chain::object_static_prototypes_maybe_nonempty()
         && crate::object::prototype_chain::residual_prototype_owner_type(
-            (*header_from_user_ptr(old_user as *const u8)).obj_type,
+            (*relocating_header).obj_type,
         )
+        // #10362: the per-OWNER half, same bit and same proof as the trace
+        // path's. `_reserved` rides every relocation by construction — all four
+        // callers copy it before calling here and
+        // `assert_relocation_copied_the_header` enforces that — so the bit is
+        // already correct at this point and needs no transfer of its own.
+        && crate::object::prototype_chain::residual_entry_possible_for(relocating_header)
     {
         transfer_residual_prototype(old_user as usize, new_user as usize);
     }

crates/perry-runtime/src/gc/layout_slot_visit.rs

@@ -167,6 +167,12 @@ pub(super) unsafe fn visit_gc_rewrite_slot_descriptors(
     // arms, so no arm's early return can skip it.
     if crate::object::prototype_chain::object_static_prototypes_maybe_nonempty()
         && crate::object::prototype_chain::residual_prototype_owner_type(obj_type)
+        // #10362: the per-OWNER half. The latch above is exact for a process
+        // that never re-prototyped a non-object and useless for one that has —
+        // it is what made a single `Object.setPrototypeOf(anArray, p)` charge
+        // every traced cell of every owner-capable kind a global mutex and a
+        // SipHash probe. This asks the owner's own header instead.
+        && crate::object::prototype_chain::residual_entry_possible_for(header)
     {
         crate::object::prototype_chain::visit_object_static_prototype_slot_mut(
             user_ptr as usize,

crates/perry-runtime/src/gc/tests/residual_prototype_relocation.rs

@@ -175,6 +175,80 @@ fn test_lazy_array_explicit_prototype_survives_a_copying_minor() {
     );
 }
 
+/// #10362 SABOTAGE: with `GC_RESIDUAL_PROTO_OWNER` never set, the per-owner
+/// gates answer "no entry here" and the collector skips the prototype edge.
+///
+/// This is the test that proves the bit is load-bearing rather than
+/// documentation. Its subject is the same one the two tests around it assert on
+/// — the entry must follow the owner and the recorded address must be rewritten
+/// — so a green run here with the bit suppressed would mean the gates it feeds
+/// are not gating anything, and A' should be withdrawn.
+///
+/// The membership assertion (`debug_assert_residual_owner_bit`) stands itself
+/// down while the sabotage is armed, so this test fails at the real verdict
+/// rather than at an assertion the sabotage itself provoked.
+#[test]
+fn test_suppressing_the_residual_owner_bit_loses_the_prototype() {
+    let _serialized = crate::array::test_serialize();
+    let _feedback = crate::typed_feedback::typed_feedback_test_lock();
+    let _latch = ArrayPrototypeLatchRestore::capture();
+    let _guard = CopyingNurseryTestGuard::new(1);
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+
+    let owner = crate::array::js_array_alloc(4) as usize;
+    let obj_type = obj_type_at(owner);
+    assert!(
+        crate::arena::pointer_in_nursery(owner) && crate::gc::gc_type_is_movable(obj_type),
+        "premise: a nursery owner of a movable kind"
+    );
+    assert_ne!(
+        obj_type, GC_TYPE_OBJECT,
+        "premise: the subject must be a kind the bit actually gates"
+    );
+    js_shadow_slot_set(0, ptr_bits(owner));
+
+    let sabotage = crate::object::prototype_chain::residual_proto_bit_sabotage::Guard::arm();
+    let proto = marked_prototype();
+    crate::object::js_object_set_prototype_of(
+        f64::from_bits(ptr_bits(owner)),
+        f64::from_bits(ptr_bits(proto)),
+    );
+    let owner = (js_shadow_slot_get(0) & POINTER_MASK) as usize;
+    assert_eq!(
+        crate::object::prototype_chain::object_static_prototype(owner),
+        Some(ptr_bits(proto)),
+        "premise: the entry is recorded even with the bit suppressed — the \
+         sabotage removes the PROOF, not the entry"
+    );
+    unsafe {
+        let header =
+            (owner as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader;
+        assert_eq!(
+            (*header)._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER,
+            0,
+            "premise: the sabotage really did suppress the bit"
+        );
+    }
+
+    let trace = collect_minor_trace(GcTriggerKind::Direct);
+    assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false);
+    let owner_after = (js_shadow_slot_get(0) & POINTER_MASK) as usize;
+    assert_ne!(owner_after, owner, "premise: the owner must actually move");
+
+    let recorded = crate::object::prototype_chain::object_static_prototype(owner_after);
+    forget_owners(&[owner, owner_after]);
+    js_shadow_slot_set(0, 0);
+    drop(sabotage);
+
+    assert!(
+        recorded.is_none(),
+        "the bit is not load-bearing: with `GC_RESIDUAL_PROTO_OWNER` suppressed the \
+         entry still followed its owner, so the per-owner gates in \
+         `gc/layout_slot_visit.rs` and `gc/layout/transfer.rs` are not gating \
+         anything. A' is documentation — withdraw it."
+    );
+}
+
 /// Every movable receiver kind that keeps its prototype in the residual
 /// registry, with the prototype held by NOTHING but that entry. One copying
 /// minor has to rekey the entry, retain the prototype through it, and rewrite

crates/perry-runtime/src/gc/types.rs

@@ -1294,6 +1294,44 @@ pub const OBJ_FLAG_TYPED_ARRAY_PROTO: u16 = 0x100;
 /// `JSValue` slots. This is only meaningful for `GC_TYPE_ARRAY`; object
 /// flags share the same `_reserved` word but never inspect this bit.
 pub(crate) const GC_ARRAY_RAW_F64_LAYOUT: u16 = 0x80;
+/// #10362: this cell owns an entry in the residual static-prototype registry
+/// (`object::prototype_chain`) — i.e. an `Object.setPrototypeOf` whose receiver
+/// `meta_capable_object` turned away, so the prototype could not go in a meta
+/// record and went into the address-keyed table instead.
+///
+/// The registry's readers used to ask only the process-global
+/// `OBJECT_PROTOTYPES_NONEMPTY` latch, which is exact for a process that has
+/// never re-prototyped a non-object and useless for one that has: a single
+/// `Object.setPrototypeOf(anArray, p)` made EVERY traced cell of every
+/// owner-capable kind take the registry's global mutex and a SipHash probe, and
+/// every relocation of one call the rekey hook. Measured on a 200k-array
+/// fixture: 400,000 armed hook calls from three lines of setup, against 155
+/// unarmed. This bit answers the same question per OWNER, so an armed process
+/// pays only for the cells that actually have an entry.
+///
+/// Bit 6, shared with `OBJ_FLAG_NULL_PROTO` exactly as bits 7..12 are already
+/// shared between the OBJECT and ARRAY namespaces, and disjoint from it by
+/// `obj_type`: `OBJ_FLAG_NULL_PROTO` has one setter,
+/// `js_object_alloc_null_proto`, which returns `*mut ObjectHeader`, and every
+/// reader of it is behind an `obj_type == GC_TYPE_OBJECT` guard (audited:
+/// `field_get_set/accessors.rs`, `field_get_set/for_in_stable.rs`,
+/// `field_set_by_name.rs`, `field_set_by_name/tail.rs` via `object_is_regular`,
+/// `builtins/formatting.rs` via both callers' `gc_type` dispatch,
+/// `builtins/formatting/prototype_equality.rs` via `heap_object_addr`,
+/// `native_call_method/object_proto.rs` via `object_ptr_from_value`).
+/// So this bit is **only meaningful for `obj_type != GC_TYPE_OBJECT`**, and a
+/// `GC_TYPE_OBJECT` owner that reaches the registry anyway keeps the
+/// latch-only gate (see `residual_entry_possible_for`).
+///
+/// Set-only, like `GC_ARRAY_NAMED_PROPS` and for the same reason: an entry is
+/// never deleted while its owner lives (a second `setPrototypeOf` overwrites
+/// the same key; the prune only removes DEAD owners, whose header is gone; the
+/// two rekey paths remove-then-insert and the entry survives). So the error is
+/// on the safe side by construction — a stale-set bit costs one probe that the
+/// map answers `None` to, while the dangerous direction (entry present, bit
+/// absent) has no code path that can produce it, because the only writer of the
+/// entry is also the only writer of the bit, under one lock.
+pub(crate) const GC_RESIDUAL_PROTO_OWNER: u16 = 0x40;
 /// Array was synthesized for a function's `arguments` binding. This is only
 /// meaningful for `GC_TYPE_ARRAY`; it lets `util.types.isArgumentsObject`
 /// distinguish Perry's internal `arguments` arrays from user rest arrays.

crates/perry-runtime/src/object/prototype_chain.rs

@@ -160,6 +160,130 @@ pub(crate) fn any_user_prototype_override() -> bool {
     USER_PROTO_OVERRIDE_EVER.load(Ordering::Acquire)
 }
 
+/// #10362: mark `obj_ptr`'s own header as an owner in the residual registry.
+///
+/// Called under the registry lock and BEFORE the insert, the same discipline
+/// `OBJECT_PROTOTYPES_NONEMPTY` uses one line below: the proof is published
+/// before the fact it guards, so a reader that can observe the entry already
+/// observes the bit.
+///
+/// Silently does nothing for a `GC_TYPE_OBJECT` owner. Such an owner normally
+/// never reaches the registry at all (`meta_capable_object` takes it), but it
+/// can when that function turns it away for a non-type reason — and bit 6 means
+/// `OBJ_FLAG_NULL_PROTO` there, so it must not be reused. Those owners keep the
+/// latch-only gate, which is what every owner had before this change.
+unsafe fn set_residual_proto_owner_bit(obj_ptr: usize) {
+    #[cfg(test)]
+    if residual_proto_bit_sabotage::suppressed() {
+        return;
+    }
+    let Some(header) = crate::value::addr_class::try_read_gc_header(obj_ptr) else {
+        return;
+    };
+    if header.obj_type == crate::gc::GC_TYPE_OBJECT {
+        return;
+    }
+    let header = (obj_ptr as *mut u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader;
+    (*header)._reserved |= crate::gc::GC_RESIDUAL_PROTO_OWNER;
+}
+
+/// Can the cell at `header` own a residual-prototype entry, judged from its own
+/// header rather than from the process-global latch?
+///
+/// This is the per-owner half of the registry's gate. Callers keep asking
+/// [`object_static_prototypes_maybe_nonempty`] FIRST — it is one byte load and
+/// false for any process that never re-prototyped a non-object — and ask this
+/// second, which is what stops an ARMED process paying per traced cell.
+///
+/// Conservative for `GC_TYPE_OBJECT`: see `set_residual_proto_owner_bit`.
+///
+/// # Safety
+///
+/// `header` is a readable `GcHeader` of a live allocation.
+#[inline]
+pub(crate) unsafe fn residual_entry_possible_for(header: *const crate::gc::GcHeader) -> bool {
+    if (*header).obj_type == crate::gc::GC_TYPE_OBJECT {
+        return true;
+    }
+    (*header)._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER != 0
+}
+
+/// The invariant the collector's gates rest on: a LIVE non-object owner that
+/// has an entry in the registry carries the bit.
+///
+/// Asserted in test and debug builds — including `cargo test --release`, how the
+/// GC suites run — for the same reason
+/// `gc::layout::transfer::assert_relocation_copied_the_header` is: a test proves
+/// today's code, an assertion proves tomorrow's. A future path that inserts an
+/// entry without the bit would make every collector gate skip that owner's
+/// prototype edge, which is #10493's bug exactly — correct before a collection,
+/// wrong after, exit code 0 and no warning.
+///
+/// Only this direction is an invariant. The reverse (bit set implies an entry)
+/// is deliberately NOT asserted: the bit is set-only, and the two rekey paths
+/// remove-then-insert with the lock released in between, so a bit without an
+/// entry is a legal transient and a benign steady state.
+#[inline]
+pub(crate) unsafe fn debug_assert_residual_owner_bit(obj_ptr: usize) {
+    #[cfg(any(test, debug_assertions))]
+    {
+        #[cfg(test)]
+        if residual_proto_bit_sabotage::suppressed() {
+            return;
+        }
+        if let Some(header) = crate::value::addr_class::try_read_gc_header(obj_ptr) {
+            if header.obj_type != crate::gc::GC_TYPE_OBJECT {
+                assert!(
+                    header._reserved & crate::gc::GC_RESIDUAL_PROTO_OWNER != 0,
+                    "residual prototype registry: owner {obj_ptr:#x} (obj_type {}) has an \
+                     entry but not `GC_RESIDUAL_PROTO_OWNER`, so every collector gate will \
+                     skip its prototype edge — the prototype is neither retained nor \
+                     rewritten (#10493's failure mode)",
+                    header.obj_type
+                );
+            }
+        }
+    }
+    #[cfg(not(any(test, debug_assertions)))]
+    {
+        let _ = obj_ptr;
+    }
+}
+
+/// Test-only sabotage for [`set_residual_proto_owner_bit`]: the bit is never
+/// set, so every per-owner gate falls back to "no entry here" and the collector
+/// skips the prototype edge.
+///
+/// Both tests in `gc/tests/residual_prototype_relocation.rs` MUST fail while
+/// this is armed. If they pass, the bit is not load-bearing and is
+/// documentation — the failure mode CLAUDE.md calls "a gate that cannot fail".
+#[cfg(test)]
+pub(crate) mod residual_proto_bit_sabotage {
+    use std::cell::Cell;
+
+    thread_local! {
+        static SUPPRESSED: Cell<bool> = const { Cell::new(false) };
+    }
+
+    pub(crate) fn suppressed() -> bool {
+        SUPPRESSED.with(Cell::get)
+    }
+
+    pub(crate) struct Guard(bool);
+
+    impl Guard {
+        pub(crate) fn arm() -> Self {
+            Self(SUPPRESSED.with(|s| s.replace(true)))
+        }
+    }
+
+    impl Drop for Guard {
+        fn drop(&mut self) {
+            SUPPRESSED.with(|s| s.set(self.0));
+        }
+    }
+}
+
 fn get_object_prototypes() -> &'static Mutex<HashMap<usize, u64>> {
     OBJECT_PROTOTYPES.get_or_init(|| Mutex::new(HashMap::new()))
 }
@@ -325,6 +449,9 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, link_kind:
         // The publish property is unchanged: a reader that sees `true` takes
         // the lock and therefore sees whatever the writer committed.
         OBJECT_PROTOTYPES_NONEMPTY.store(true, Ordering::Release);
+        // #10362: the per-OWNER half of the same proof, published under the
+        // same lock and before the same insert, for the same reason.
+        unsafe { set_residual_proto_owner_bit(obj_ptr) };
         let slot = map.entry(obj_ptr).or_insert(0);
         *slot = proto_bits;
         slot_addr = slot as *mut u64 as usize;
@@ -360,10 +487,17 @@ pub fn object_static_prototype(obj_ptr: usize) -> Option<u64> {
     if !OBJECT_PROTOTYPES_NONEMPTY.load(Ordering::Acquire) {
         return None;
     }
-    get_object_prototypes()
+    let recorded = get_object_prototypes()
         .lock()
         .ok()
-        .and_then(|map| map.get(&obj_ptr).copied())
+        .and_then(|map| map.get(&obj_ptr).copied());
+    // #10362: the invariant every collector gate rests on, checked on the read
+    // paths that are NOT gated by the bit — asserting it inside a bit-gated
+    // path would be vacuous.
+    if recorded.is_some() {
+        unsafe { debug_assert_residual_owner_bit(obj_ptr) };
+    }
+    recorded
 }
 
 /// Look up the residual prototype registry for a caller that has already
@@ -383,10 +517,14 @@ pub(crate) fn object_static_prototype_known_non_meta(obj_ptr: usize) -> Option<u
     if !OBJECT_PROTOTYPES_NONEMPTY.load(Ordering::Acquire) {
         return None;
     }
-    get_object_prototypes()
+    let recorded = get_object_prototypes()
         .lock()
         .ok()
-        .and_then(|map| map.get(&obj_ptr).copied())
+        .and_then(|map| map.get(&obj_ptr).copied());
+    if recorded.is_some() {
+        unsafe { debug_assert_residual_owner_bit(obj_ptr) };
+    }
+    recorded
 }
 
 #[inline]