From 99f1f10630d168f34135051ae8859ad4efc7edaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 06:17:38 +0200 Subject: [PATCH 1/2] perf(gc): make the "nothing due" GC check cheap on safepoint polls and trigger checks Runtime safepoint polls (regex quanta, the microtask pump, the event loop), gc_malloc and the other gc_check_trigger callers spent hundreds to thousands of instructions answering "is a collection due?" when nothing was. Three changes, each leaving every collection decision unchanged: - Runtime polls no longer build a JsGcStepResult. The budgeted step returns a debt-free GcStepReport; the FFI and test entry points attach the GcDebtSnapshot after the step returns, which reads the same values. The cycle start/step machinery moved out of line so a no-trigger poll does not pay its multi-kilobyte frame. - copying_from_space_in_use_bytes() is O(1) between layout changes: the bytes outside Eden's current block are cached, keyed on the heap generation (every reset, detach, evacuation and survivor flip runs inside a HeapChange scope), and every move of an arena's current block goes through Arena::set_current, which invalidates the cache. Debug builds compare every cached answer with the block walk. - gc_check_trigger evaluates the due trigger once instead of up to three times, reusing the answer unless it came from the #10169 one-shot leaf priority; the young cap reuses the old-gen pressure the due trigger already read (debug cross-checked) and tests the census-seeded flag first. --- crates/perry-runtime/src/arena/block.rs | 22 +- crates/perry-runtime/src/arena/from_space.rs | 295 ++++++++++++++++++ crates/perry-runtime/src/arena/mod.rs | 1 + crates/perry-runtime/src/arena/promote.rs | 5 +- crates/perry-runtime/src/arena/quarantine.rs | 7 +- crates/perry-runtime/src/arena/reset.rs | 42 +-- .../perry-runtime/src/gc/heap_generation.rs | 9 +- crates/perry-runtime/src/gc/mod.rs | 2 + crates/perry-runtime/src/gc/policy.rs | 227 +++++++++++--- crates/perry-runtime/src/gc/tenuring.rs | 8 + .../src/gc/tests/young_leaf_route.rs | 75 ++++- crates/perry-runtime/src/lib.rs | 2 +- .../perry-runtime/src/promise/microtasks.rs | 2 +- .../perry-runtime/src/regex/perex_runtime.rs | 2 +- scripts/gc_runtime_root_holders.json | 12 +- 15 files changed, 627 insertions(+), 84 deletions(-) create mode 100644 crates/perry-runtime/src/arena/from_space.rs diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index 96e64195d1..48b916d831 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -576,6 +576,22 @@ impl Arena { /// Only used for the non-Eden regions — Eden must stay eager /// because `js_inline_arena_state` hands its current block to /// codegen's inline bump allocator at thread start. + /// Point allocation at `blocks[idx]`. + /// + /// Every move of `current` goes through here, because the cached young + /// occupancy (`arena/from_space.rs`) assumes that between two reads only + /// the current Eden block's offset changed unless the heap generation + /// moved. The allocator's block switches happen outside any `HeapChange` + /// scope, so they must retire that cache themselves; the cache key does not + /// include the block index. Invalidating for every arena, not just Eden, + /// keeps the rule free of a space test; a switch costs one thread-local + /// store and happens once per block. + #[inline] + pub(crate) fn set_current(&mut self, idx: usize) { + self.current = idx; + super::from_space::invalidate_sealed_young_bytes(); + } + fn new_lazy(generation: HeapGeneration, space: HeapSpace) -> Self { Arena { blocks: vec![ArenaBlock { @@ -685,7 +701,7 @@ impl Arena { self.blocks.len() - 1 } }; - self.current = new_idx; + self.set_current(new_idx); ARENA_TOTAL_BYTES.with(|t| t.set(t.get() + fresh_size)); } @@ -757,7 +773,7 @@ impl Arena { continue; } if let Some(ptr) = self.try_block_alloc(i, size, align) { - self.current = i; + self.set_current(i); // Resync inline state to the new current block. self.resync_inline_to_current(); return Some(ptr); @@ -797,7 +813,7 @@ impl Arena { } if let Some(ptr) = self.try_block_alloc_excluding_pages(i, size, align, excluded_pages) { - self.current = i; + self.set_current(i); self.resync_inline_to_current(); return ptr; } diff --git a/crates/perry-runtime/src/arena/from_space.rs b/crates/perry-runtime/src/arena/from_space.rs new file mode 100644 index 0000000000..9ea4c8e1b2 --- /dev/null +++ b/crates/perry-runtime/src/arena/from_space.rs @@ -0,0 +1,295 @@ +//! Young-generation occupancy without a block walk. +//! +//! `copying_from_space_in_use_bytes()` is the basis of the scavenge nursery +//! cap, and `gc_budgeted_due_trigger()` reads it on every runtime safepoint +//! poll and every `gc_malloc`. Summing `block.offset` over every Eden block and +//! every active-survivor block made each of those reads O(blocks), and the +//! young generation can hold hundreds of blocks once the cap scales with the +//! tenured set. +//! +//! # What changes between two reads +//! +//! Split the sum into the Eden block allocation is currently bumping +//! (`blocks[current]`) and everything else, the *sealed* bytes. Between two +//! reads the sealed bytes can change in only three ways: +//! +//! 1. **A free or move.** Every reset, detach, release, evacuation and survivor +//! flip runs inside a [`HeapChange`](crate::gc::heap_generation::HeapChange) +//! scope, which advances the heap generation when it opens and again when it +//! closes. The funnel is enforced: the reset and evacuation primitives call +//! `debug_assert_heap_change_open`. +//! 2. **The allocator moves `current`.** Filling a block and continuing in +//! another one (a fresh block, a reused tombstone slot, or a partly-used +//! block with room for a smaller request) happens outside any scope. Every +//! such move goes through [`Arena::set_current`], which invalidates the +//! cache. Comparing the index alone would not do: `current` can leave +//! block 3 for block 5 and come back to block 3 once block 5 is full. +//! 3. **The active survivor space changes.** That is a flip, so it is case 1, +//! and the index is part of the key anyway. +//! +//! Survivor allocation fills the *inactive* space, which is not from-space +//! until the flip, and the flip happens inside the copying minor's scope. +//! +//! So the cache holds the sealed bytes keyed on the heap generation, and a read +//! is `sealed + blocks[current].offset`. Eden's `current` and the active +//! survivor index are deliberately not part of the key: every move of either is +//! already covered by rule 2 or rule 1, and each extra key field is one more +//! checked thread-local lookup on a path the due check takes on every poll. +//! Nothing is stored while a scope is open, because the generation does not +//! move inside one; a value stored before the scope opened stops matching when +//! it opens. +//! +//! Debug builds compare every cached answer with the walk, so a mutation site +//! that bypasses both rules fails the test that reaches it rather than skewing +//! the nursery cap. + +use super::*; + +#[derive(Clone, Copy)] +struct SealedYoungBytes { + valid: bool, + heap_generation: u64, + /// Eden bytes outside the current block, plus every byte of the active + /// survivor space. + sealed: usize, +} + +impl SealedYoungBytes { + const INVALID: Self = Self { + valid: false, + heap_generation: 0, + sealed: 0, + }; +} + +crate::perry_thread_local! { + static SEALED_YOUNG_BYTES: Cell = const { Cell::new(SealedYoungBytes::INVALID) }; +} + +#[cfg(test)] +crate::perry_thread_local! { + /// Fault injection for the invalidation test: while set, `Arena::set_current` + /// leaves the cache alone. + static SKIP_SET_CURRENT_INVALIDATION: Cell = const { Cell::new(false) }; +} + +/// Forget the cached sealed bytes. Called when an arena moves `current`. +#[inline] +pub(crate) fn invalidate_sealed_young_bytes() { + #[cfg(test)] + if SKIP_SET_CURRENT_INVALIDATION.with(Cell::get) { + return; + } + SEALED_YOUNG_BYTES.with(|cache| cache.set(SealedYoungBytes::INVALID)); +} + +/// Bytes currently allocated in Eden plus the active survivor from-space. +#[inline] +pub(crate) fn copying_from_space_in_use_bytes() -> usize { + let current_bytes = synced_current_eden_block_bytes(); + let generation = crate::gc::heap_generation::heap_generation(); + let cached = SEALED_YOUNG_BYTES.with(Cell::get); + if cached.valid && cached.heap_generation == generation { + let bytes = cached.sealed + current_bytes; + debug_assert_eq!( + bytes, + copying_from_space_in_use_bytes_walked(), + "cached from-space occupancy drifted from the block walk: a young block offset \ + or Eden's `current` changed outside a HeapChange scope without \ + `Arena::set_current` (see arena/from_space.rs)" + ); + return bytes; + } + walk_and_store(generation, current_bytes) +} + +/// `sync_inline_arena_state`, then the current Eden block's offset. The same +/// write and the same sampling note, through the hot-TLS addresses rather than +/// two `thread_local!` resolutions. +#[inline(always)] +fn synced_current_eden_block_bytes() -> usize { + // SAFETY: this thread's inline state and nursery arena; no `&mut` borrow of + // either is live across this call. + unsafe { + let inline = &*hot_inline_state(); + let eden = &mut *hot_arena(); + let current = eden.current; + if !inline.data.is_null() { + super::alloc_sample::note_inline_sync(eden.blocks[current].offset, inline.offset); + eden.blocks[current].offset = inline.offset; + } + eden.blocks[current].offset + } +} + +#[cold] +#[inline(never)] +fn walk_and_store(generation: u64, current_bytes: usize) -> usize { + let bytes = copying_from_space_in_use_bytes_walked(); + if !crate::gc::heap_generation::heap_change_open() { + SEALED_YOUNG_BYTES.with(|cache| { + cache.set(SealedYoungBytes { + valid: true, + heap_generation: generation, + sealed: bytes - current_bytes, + }) + }); + } + bytes +} + +/// The O(blocks) sum the cache stands in for. The source of truth. +pub(crate) fn copying_from_space_in_use_bytes_walked() -> usize { + sync_inline_arena_state(); + let eden = ARENA.with(|arena| { + let arena = unsafe { &*arena.get() }; + arena.blocks.iter().map(|b| b.offset).sum::() + }); + let active = ACTIVE_SURVIVOR.with(|active| active.get()); + let survivor = with_survivor_arena(active, |arena| { + arena.blocks.iter().map(|b| b.offset).sum::() + }); + eden + survivor +} + +#[cfg(test)] +mod tests { + use super::*; + + struct SkipInvalidation; + + impl SkipInvalidation { + fn new() -> Self { + SKIP_SET_CURRENT_INVALIDATION.with(|skip| skip.set(true)); + Self + } + } + + impl Drop for SkipInvalidation { + fn drop(&mut self) { + SKIP_SET_CURRENT_INVALIDATION.with(|skip| skip.set(false)); + invalidate_sealed_young_bytes(); + } + } + + /// The cached answer if the cache matches, without the debug cross-check, + /// so the fault-injection test can observe a stale value instead of + /// panicking inside the read. + fn cache_hit() -> Option { + let current_bytes = synced_current_eden_block_bytes(); + let cached = SEALED_YOUNG_BYTES.with(Cell::get); + (cached.valid && cached.heap_generation == crate::gc::heap_generation::heap_generation()) + .then(|| cached.sealed + current_bytes) + } + + fn eden_current() -> usize { + ARENA.with(|a| unsafe { (*a.get()).current }) + } + + fn eden_block(idx: usize) -> (usize, usize) { + ARENA.with(|a| { + let arena = unsafe { &*a.get() }; + let block = &arena.blocks[idx]; + (block.offset, block.size) + }) + } + + /// Prime the cache on Eden's only block with 64 bytes of headroom, continue + /// in a fresh block, fill it, and return to the first block through the + /// allocator's forward scan: `current` ends at the index it was primed at, + /// while a whole block of bytes landed elsewhere. Every step asserts where + /// the allocator went, so a change in allocation policy fails the test + /// instead of making it vacuous. + fn return_to_the_primed_block() { + let _no_gc = crate::gc::GcSuppressScope::new(); + // libtest runs each test on a fresh thread, so this is a fresh arena. + assert_eq!( + ARENA.with(|a| unsafe { (*a.get()).blocks.len() }), + 1, + "fixture needs a fresh thread-local Eden" + ); + let start = eden_current(); + let (used, size) = eden_block(start); + let _ = crate::arena::arena_alloc(size - used - 64, 8); + assert_eq!(eden_block(start).0, size - 64); + + let primed = copying_from_space_in_use_bytes(); + assert_eq!( + cache_hit(), + Some(primed), + "the first read must prime the cache" + ); + + // 128 bytes do not fit the 64-byte headroom: a fresh block is installed. + let _ = crate::arena::arena_alloc(128, 8); + let other = eden_current(); + assert_ne!(other, start); + let (other_used, other_size) = eden_block(other); + let _ = crate::arena::arena_alloc(other_size - other_used, 8); + assert_eq!( + eden_block(other).0, + other_size, + "the fresh block must be full" + ); + + // Only the primed block can serve 32 bytes now. + let _ = crate::arena::arena_alloc(32, 8); + assert_eq!( + eden_current(), + start, + "the forward scan must return to the primed block" + ); + assert!(copying_from_space_in_use_bytes_walked() >= primed + other_size); + } + + #[test] + fn cached_occupancy_matches_the_walk_after_returning_to_the_primed_block() { + return_to_the_primed_block(); + assert_eq!( + cache_hit(), + None, + "set_current must have invalidated the cache" + ); + assert_eq!( + copying_from_space_in_use_bytes(), + copying_from_space_in_use_bytes_walked() + ); + } + + /// Fault injection for the invalidation in `Arena::set_current`: without it + /// the same sequence leaves a cache that matches every key and is short by + /// the fresh block's bytes. + #[test] + fn without_set_current_invalidation_the_cache_goes_stale() { + let _skip = SkipInvalidation::new(); + return_to_the_primed_block(); + let walked = copying_from_space_in_use_bytes_walked(); + let stale = cache_hit().expect("with invalidation skipped the primed entry still matches"); + assert!( + stale < walked, + "the stale cache must miss the fresh block's bytes (stale {stale}, walked {walked})" + ); + } + + /// The other half of the key: a collection opens a `HeapChange`, and the + /// generation it advances retires the cached entry. + #[test] + fn a_heap_change_scope_retires_the_cached_entry() { + let _no_gc = crate::gc::GcSuppressScope::new(); + let _ = crate::arena::arena_alloc(256, 8); + let primed = copying_from_space_in_use_bytes(); + assert_eq!(cache_hit(), Some(primed)); + { + let _scope = crate::gc::heap_generation::HeapChange::begin( + crate::gc::heap_generation::HeapChangeKind::Sweep, + ); + assert_eq!(cache_hit(), None); + // Reads inside the scope walk and store nothing. + assert_eq!(copying_from_space_in_use_bytes(), primed); + assert_eq!(cache_hit(), None); + } + assert_eq!(cache_hit(), None); + assert_eq!(copying_from_space_in_use_bytes(), primed); + assert_eq!(cache_hit(), Some(primed)); + } +} diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 1fc2224482..785f7dc4d1 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -12,6 +12,7 @@ pub(crate) mod alloc_sample; mod allocators; mod block; mod construction; +mod from_space; pub(crate) use construction::ConstructionBatch; mod inline; mod page_meta; diff --git a/crates/perry-runtime/src/arena/promote.rs b/crates/perry-runtime/src/arena/promote.rs index fcfacf6442..7d5bdbd3a3 100644 --- a/crates/perry-runtime/src/arena/promote.rs +++ b/crates/perry-runtime/src/arena/promote.rs @@ -587,11 +587,12 @@ fn reset_young_after_promotion() { if arena.blocks.iter().all(|block| block.data.is_null()) { arena.install_fresh_block(BLOCK_SIZE); } - arena.current = arena + let first_live = arena .blocks .iter() .position(|block| !block.data.is_null()) .unwrap_or(0); + arena.set_current(first_live); INLINE_STATE.with(|s| { let inline = &mut *s.get(); if !inline.data.is_null() { @@ -610,7 +611,7 @@ fn reset_young_after_promotion() { block.offset = 0; block.dead_cycles = 0; } - arena.current = 0; + arena.set_current(0); }); } let active = ACTIVE_SURVIVOR.with(|active| active.get()); diff --git a/crates/perry-runtime/src/arena/quarantine.rs b/crates/perry-runtime/src/arena/quarantine.rs index 37986e7e06..30d0571532 100644 --- a/crates/perry-runtime/src/arena/quarantine.rs +++ b/crates/perry-runtime/src/arena/quarantine.rs @@ -580,11 +580,12 @@ pub(crate) fn copying_quarantine_from_spaces_and_flip() -> ArenaResetStats { }); with_survivor_arena_mut(active, |arena| { - arena.current = arena + let first_live = arena .blocks .iter() .position(|block| !block.data.is_null()) .unwrap_or(0); + arena.set_current(first_live); }); ACTIVE_SURVIVOR.with(|active_cell| active_cell.set(1 - active)); @@ -619,7 +620,7 @@ unsafe fn detach_used_blocks(arena: &mut Arena) -> Vec<(*mut u8, usize, usize)> block.offset = 0; block.dead_cycles = 0; } - arena.current = 0; + arena.set_current(0); detached } @@ -654,7 +655,7 @@ unsafe fn ensure_usable_current_block(arena: &mut Arena) { .iter() .position(|block| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); return; } arena.install_fresh_block(BLOCK_SIZE); diff --git a/crates/perry-runtime/src/arena/reset.rs b/crates/perry-runtime/src/arena/reset.rs index faf9525426..e643f869a0 100644 --- a/crates/perry-runtime/src/arena/reset.rs +++ b/crates/perry-runtime/src/arena/reset.rs @@ -22,7 +22,7 @@ pub fn arena_reset_all_blocks_to_zero() { block.clear_object_starts(); block.offset = 0; } - arena.current = 0; + arena.set_current(0); // Free list is now invalid (all entries point into reset blocks). crate::gc::ARENA_FREE_LIST.with(|fl| fl.borrow_mut().clear()); crate::gc::ARENA_FREE_LIST_NONEMPTY.with(|c| c.set(false)); @@ -110,7 +110,7 @@ fn reset_region_to_zero(arena: &mut Arena) -> (usize, usize) { if arena.generation == HeapGeneration::Old { old_gen_in_use_bytes_sub(reusable_bytes); } - arena.current = 0; + arena.set_current(0); (reset_blocks, reusable_bytes) } @@ -129,18 +129,8 @@ pub(crate) fn copying_active_survivor_in_use_bytes() -> usize { } /// Bytes currently allocated in Eden plus the active survivor from-space. -pub(crate) fn copying_from_space_in_use_bytes() -> usize { - sync_inline_arena_state(); - let eden = ARENA.with(|arena| { - let arena = unsafe { &*arena.get() }; - arena.blocks.iter().map(|b| b.offset).sum::() - }); - let active = ACTIVE_SURVIVOR.with(|active| active.get()); - let survivor = with_survivor_arena(active, |arena| { - arena.blocks.iter().map(|b| b.offset).sum::() - }); - eden + survivor -} +/// O(1) between layout changes; see `arena/from_space.rs`. +pub(crate) use super::from_space::copying_from_space_in_use_bytes; /// #7901: is `block_idx` inside the copying collector's from-space (Eden plus /// the ACTIVE survivor semispace)? The inactive semispace is to-space and is @@ -501,7 +491,7 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats { // `arena.current` where it was — the next `Arena::alloc` slow // path will tombstone-reuse a slot and update `current` then. if !arena.blocks[new_current].data.is_null() { - arena.current = new_current; + arena.set_current(new_current); } let _ = (n_live, n_total); INLINE_STATE.with(|s| { @@ -817,7 +807,7 @@ impl ArenaResetEmptyBlocksState { .map(|block| !block.data.is_null()) .unwrap_or(false) { - arena.current = new_current; + arena.set_current(new_current); } INLINE_STATE.with(|s| { let inline = &mut *s.get(); @@ -980,7 +970,7 @@ impl SurvivorArenaReclaimState { .enumerate() .find(|(_, block)| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); } else if arena .blocks .get(arena.current) @@ -993,7 +983,7 @@ impl SurvivorArenaReclaimState { .enumerate() .find(|(_, block)| !block.data.is_null()) { - arena.current = idx; + arena.set_current(idx); } } }); @@ -1297,7 +1287,7 @@ impl OldArenaReclaimDeadBlocksState { .enumerate() .find(|(_, block)| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); } else if arena .blocks .get(arena.current) @@ -1310,7 +1300,7 @@ impl OldArenaReclaimDeadBlocksState { .enumerate() .find(|(_, block)| !block.data.is_null()) { - arena.current = idx; + arena.set_current(idx); } } }); @@ -1382,7 +1372,7 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes .enumerate() .find(|(_, block)| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); } else if arena .blocks .get(arena.current) @@ -1395,7 +1385,7 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes .enumerate() .find(|(_, block)| !block.data.is_null()) { - arena.current = idx; + arena.set_current(idx); } } } @@ -1482,7 +1472,7 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks( .enumerate() .find(|(_, block)| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); } else if arena .blocks .get(arena.current) @@ -1495,7 +1485,7 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks( .enumerate() .find(|(_, block)| !block.data.is_null()) { - arena.current = idx; + arena.set_current(idx); } } } @@ -1579,7 +1569,7 @@ fn reclaim_dead_survivor_arena_blocks( .enumerate() .find(|(_, block)| !block.data.is_null() && block.offset == 0) { - arena.current = idx; + arena.set_current(idx); } else if arena .blocks .get(arena.current) @@ -1592,7 +1582,7 @@ fn reclaim_dead_survivor_arena_blocks( .enumerate() .find(|(_, block)| !block.data.is_null()) { - arena.current = idx; + arena.set_current(idx); } } } diff --git a/crates/perry-runtime/src/gc/heap_generation.rs b/crates/perry-runtime/src/gc/heap_generation.rs index 4f91460663..18841477b3 100644 --- a/crates/perry-runtime/src/gc/heap_generation.rs +++ b/crates/perry-runtime/src/gc/heap_generation.rs @@ -69,7 +69,6 @@ crate::perry_thread_local! { /// This thread's current heap generation. #[inline] -#[cfg_attr(not(feature = "regex-engine"), allow(dead_code))] pub(crate) fn heap_generation() -> u64 { HEAP_GENERATION.with(Cell::get) } @@ -113,6 +112,14 @@ impl Drop for HeapChange { } } +/// Is a [`HeapChange`] scope open on this thread? A cache keyed on the heap +/// generation must not store while one is, because the generation does not +/// advance inside a scope. +#[inline] +pub(crate) fn heap_change_open() -> bool { + OPEN_HEAP_CHANGES.try_with(Cell::get).unwrap_or(1) > 0 +} + /// Called by every primitive that frees or moves heap memory. #[inline] #[track_caller] diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index cd2027c9ff..6e05e3af9f 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -38,7 +38,9 @@ pub use types::*; mod json_defer; mod policy; pub(crate) use json_defer::JsonParseAllocation; +#[cfg(test)] pub(crate) use policy::gc_runtime_safepoint; +pub(crate) use policy::gc_runtime_safepoint_poll; pub(crate) use policy::note_young_leaf_born_old; /// The one writer of `GC_SAFEPOINT_PENDING` — it also keeps the poll's global /// arming shadow in step. See `gc/poll_arm.rs`. diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index 6a772c0b14..7051fd8d43 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -105,6 +105,16 @@ pub(super) fn next_arena_trigger_base() -> usize { /// (near-zero infant mortality), a saturated survivor space, and 1427 /// collections for a run that allocates ~1.4 GB. pub(super) fn young_scavenge_cap_due() -> bool { + young_scavenge_cap_due_with_old_reclaimable(old_gen_reclaimable_pressure_bytes) +} + +/// [`young_scavenge_cap_due`] with the old-gen reclaimable-pressure read +/// supplied by the caller. `gc_budgeted_due_trigger` has already read that +/// value for its old-reclaim arm, and nothing between that read and this one +/// touches old-gen, so it passes a closure returning the value it holds rather +/// than paying two more thread-local reads. The read happens where it always +/// did: after the census seed, which changes only the young-side factor. +fn young_scavenge_cap_due_with_old_reclaimable(old_reclaimable: impl FnOnce() -> usize) -> bool { if !nursery_cap_active() { return false; } @@ -114,10 +124,16 @@ pub(super) fn young_scavenge_cap_due() -> bool { // process, halfway to the base cap). Not while a collection is in // progress or a budgeted cycle is active — the young generation is being // rewritten then and the walk would read forwarding stubs. - if GC_FLAGS.with(|f| f.get()) & GC_FLAG_IN_ALLOC == 0 && !gc_budgeted_cycle_active() { + // + // The seeded test comes first because the seed can happen only once: in + // steady state this is one thread-local read instead of three. + if !super::tenuring::object_census_seeded() + && GC_FLAGS.with(|f| f.get()) & GC_FLAG_IN_ALLOC == 0 + && !gc_budgeted_cycle_active() + { super::tenuring::maybe_seed_object_census_from_allocation(from_space_in_use); } - from_space_in_use >= scavenge_nursery_cap_dueness_bytes() + from_space_in_use >= scavenge_nursery_cap_dueness_bytes_with(old_reclaimable) } /// #10169: does the young generation hold at least one BASE nursery cap of @@ -142,11 +158,22 @@ pub(crate) fn young_generation_holds_a_nursery() -> bool { /// NOT feed `effective_next_arena_trigger`: this is about *dueness*, and a test /// that also moved the trigger clamp would be changing two things at once. fn scavenge_nursery_cap_dueness_bytes() -> usize { + scavenge_nursery_cap_dueness_bytes_with(old_gen_reclaimable_pressure_bytes) +} + +fn scavenge_nursery_cap_dueness_bytes_with(old_reclaimable: impl FnOnce() -> usize) -> usize { #[cfg(test)] if let Some(bytes) = GC_NURSERY_CAP_TEST_DUE_BYTES.with(Cell::get) { return bytes; } - super::tenuring::scavenge_nursery_cap_effective_bytes() + let influx_driven = super::tenuring::influx_driven_nursery_cap_bytes(); + let old_reclaimable = old_reclaimable(); + debug_assert_eq!( + old_reclaimable, + old_gen_reclaimable_pressure_bytes(), + "the old-gen pressure a due check reuses must still be the current value" + ); + super::tenuring::scavenge_nursery_cap_from(influx_driven, old_reclaimable) } #[cfg(test)] @@ -2770,6 +2797,14 @@ pub fn gc_check_trigger() { return; } + // This function asks "what is due?" up to three times on the path where + // nothing is (the old-reclaim arm, the nursery arm, the assist gate). Every + // arm that acts returns, so the state each later question sees is the + // state the first one saw, and a repeatable answer can be reused. An + // answer from the #10169 flag branch is re-evaluated, exactly as before. + let mut due_memo = DueTriggerMemo::new(); + let mut due = || due_memo.get(gc_budgeted_due_trigger_eval); + // #5476: a workload that churns *large* temporaries (>16 KB, born directly // in the old arena) grows the old generation without ever exercising the // nursery. Old-gen reclaim pressure schedules a budgeted full cycle that @@ -2824,10 +2859,7 @@ pub fn gc_check_trigger() { // on. Polls off ⇒ the precise path is reached less often ⇒ this arm fires // more often ⇒ the census counter rises. Inert, not unsound. if !gc_budgeted_cycle_active() - && matches!( - gc_budgeted_due_trigger(), - Some(BudgetedGcTrigger::OldReclaim) - ) + && matches!(due(), Some(BudgetedGcTrigger::OldReclaim)) && !GC_OLD_RECLAIM_IN_PROGRESS.with(Cell::get) { let _reentry = OldReclaimReentryGuard::enter(); @@ -2909,7 +2941,7 @@ pub fn gc_check_trigger() { || gc_moving_loop_polls_enabled() || super::roots::registered_root_scanners_block_budgeted_gc()) { - let direct_kind = match gc_budgeted_due_trigger() { + let direct_kind = match due() { // #7909: `YoungScavengeCap` is a nursery-churn trigger exactly like // `ArenaBytes` here — this arm's whole job is to route nursery // pressure to a collection that can actually reclaim it, so the two @@ -3054,7 +3086,7 @@ pub fn gc_check_trigger() { } } - if !gc_budgeted_cycle_active() && gc_budgeted_due_trigger().is_none() { + if !gc_budgeted_cycle_active() && due().is_none() { return; } @@ -3255,6 +3287,45 @@ pub(crate) fn note_young_leaf_born_old() { } pub(super) fn gc_budgeted_due_trigger() -> Option { + gc_budgeted_due_trigger_eval().0 +} + +/// Reuse of a repeatable [`gc_budgeted_due_trigger_eval`] answer across the +/// questions one `gc_check_trigger` call asks. An unrepeatable answer is +/// returned once and the next question evaluates again. +pub(super) struct DueTriggerMemo(Option>); + +impl DueTriggerMemo { + pub(super) const fn new() -> Self { + Self(None) + } + + pub(super) fn get( + &mut self, + eval: impl FnOnce() -> (Option, bool), + ) -> Option { + if let Some(due) = self.0 { + return due; + } + let (due, repeatable) = eval(); + if repeatable { + self.0 = Some(due); + } + due + } +} + +/// [`gc_budgeted_due_trigger`], plus whether evaluating it again with no state +/// changed in between is guaranteed to give the same answer. +/// +/// Only the #10169 flag branch below breaks that: it consumes the flag and +/// returns early, so a second evaluation takes the ordinary path and may +/// answer differently (for example `OldReclaim`). The #8122 census seed does +/// not break it. The seed runs inside the young-cap arm before that arm's +/// comparison, so the first evaluation already compared against the seeded +/// cap; no earlier arm reads the census and the malloc arm after it does not +/// either, and a second evaluation cannot seed again. +pub(super) fn gc_budgeted_due_trigger_eval() -> (Option, bool) { // #10169: a leaf born old under young pressure gives the nursery minor // ONE-TIME priority over old-reclaim, and only while the young generation // is still unmeasured. A young generation that a minor has already @@ -3268,16 +3339,16 @@ pub(super) fn gc_budgeted_due_trigger() -> Option { if GC_YOUNG_LEAF_BORN_OLD.with(Cell::get) { GC_YOUNG_LEAF_BORN_OLD.with(|flag| flag.set(false)); if !super::young_generation_measured_retained() && young_scavenge_cap_due() { - return Some(BudgetedGcTrigger::YoungScavengeCap); + return (Some(BudgetedGcTrigger::YoungScavengeCap), false); } } let old_pending = GC_OLD_RECLAIM_PENDING.with(Cell::get); // #6010: external Map/Set side-buffer bytes escalate to OldReclaim too. - let old_in_use = - old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); + let old_reclaimable = old_gen_reclaimable_pressure_bytes(); + let old_in_use = old_reclaimable.saturating_add(external_side_live_bytes()); let old_baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.get()); if old_pending || old_reclaim_pressure_due(old_in_use, old_baseline) { - return Some(BudgetedGcTrigger::OldReclaim); + return (Some(BudgetedGcTrigger::OldReclaim), true); } // Two separately-scoped arena arms (see `young_scavenge_cap_due` for why @@ -3286,19 +3357,21 @@ pub(super) fn gc_budgeted_due_trigger() -> Option { // only. let total = crate::arena::arena_total_bytes(); if total >= next_arena_trigger_base() { - return Some(BudgetedGcTrigger::ArenaBytes); + return (Some(BudgetedGcTrigger::ArenaBytes), true); } - if young_scavenge_cap_due() { - return Some(BudgetedGcTrigger::YoungScavengeCap); + // Old-gen is untouched since the read above: the arms in between only + // read, and the census seed walks the young generation. + if young_scavenge_cap_due_with_old_reclaimable(|| old_reclaimable) { + return (Some(BudgetedGcTrigger::YoungScavengeCap), true); } let malloc_count = malloc_object_count(); let next_malloc_trigger = GC_NEXT_MALLOC_TRIGGER.with(|c| c.get()); if malloc_count >= next_malloc_trigger { - return Some(BudgetedGcTrigger::MallocCount); + return (Some(BudgetedGcTrigger::MallocCount), true); } - None + (None, true) } /// Phase 1 of the moving-GC project: run a copying (moving) minor at a @@ -3876,6 +3949,47 @@ fn gc_start_budgeted_cycle_for_pressure(progress_kind: GcProgressKind) -> Option }) } +/// What one budgeted step decided, without the debt figures. +/// +/// The step machinery returns this; only callers that hand a +/// [`JsGcStepResult`] to someone attach the debt, through +/// [`GcStepReport::with_debt`]. The runtime's own polls (regex quanta, the +/// microtask pump, the event loop) discard the result, and a +/// `GcDebtSnapshot` costs a second evaluation of the nursery cap, the arena +/// trigger and the old-reclaim band on every poll that finds nothing due. +/// +/// Attaching the debt after the step returns reads the same values it used to +/// read inside the step: the snapshot was always the last thing a step +/// computed, and the only work between that point and the caller is dropping +/// `BudgetedGcStepGuard`, which clears a flag the snapshot does not read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct GcStepReport { + pub(crate) status: u32, + phase: u32, + collection_kind: u32, + trigger_kind: u32, + active: bool, + completed: bool, +} + +impl GcStepReport { + /// The FFI result, with the debt measured now. + pub(crate) fn with_debt(self) -> JsGcStepResult { + let debt = GcDebtSnapshot::current(); + JsGcStepResult { + status: self.status, + phase: self.phase, + collection_kind: self.collection_kind, + trigger_kind: self.trigger_kind, + active: u32::from(self.active), + completed: u32::from(self.completed), + arena_debt_bytes: debt.arena_debt_bytes, + malloc_debt_objects: debt.malloc_debt_objects, + old_reclaim_debt_bytes: debt.old_reclaim_debt_bytes, + } + } +} + fn gc_step_result( status: u32, phase: u32, @@ -3883,26 +3997,22 @@ fn gc_step_result( trigger_kind: u32, active: bool, completed: bool, -) -> JsGcStepResult { - let debt = GcDebtSnapshot::current(); - JsGcStepResult { +) -> GcStepReport { + GcStepReport { status, phase, collection_kind, trigger_kind, - active: u32::from(active), - completed: u32::from(completed), - arena_debt_bytes: debt.arena_debt_bytes, - malloc_debt_objects: debt.malloc_debt_objects, - old_reclaim_debt_bytes: debt.old_reclaim_debt_bytes, + active, + completed, } } -fn gc_idle_step_result() -> JsGcStepResult { +fn gc_idle_step_result() -> GcStepReport { gc_step_result(JS_GC_STEP_STATUS_IDLE, 0, 0, 0, false, false) } -fn gc_cycle_step_result(status: u32, cycle: &BudgetedGcCycle, completed: bool) -> JsGcStepResult { +fn gc_cycle_step_result(status: u32, cycle: &BudgetedGcCycle, completed: bool) -> GcStepReport { gc_step_result( status, cycle.state.phase().ffi_code(), @@ -3913,7 +4023,7 @@ fn gc_cycle_step_result(status: u32, cycle: &BudgetedGcCycle, completed: bool) - ) } -fn gc_budgeted_status_result() -> JsGcStepResult { +fn gc_budgeted_status_result() -> GcStepReport { if !gc_budgeted_cycle_active() { return gc_idle_step_result(); } @@ -3932,7 +4042,7 @@ fn gc_budgeted_status_result() -> JsGcStepResult { } } -fn gc_budgeted_skipped_result() -> JsGcStepResult { +fn gc_budgeted_skipped_result() -> GcStepReport { if !gc_budgeted_cycle_active() { return gc_step_result(JS_GC_STEP_STATUS_SKIPPED, 0, 0, 0, false, false); } @@ -3945,7 +4055,7 @@ fn gc_budgeted_skipped_result() -> JsGcStepResult { }) } -fn gc_finish_budgeted_cycle(mut cycle: BudgetedGcCycle) -> JsGcStepResult { +fn gc_finish_budgeted_cycle(mut cycle: BudgetedGcCycle) -> GcStepReport { let outcome = cycle .state .take_outcome() @@ -3981,7 +4091,7 @@ fn gc_finish_budgeted_cycle(mut cycle: BudgetedGcCycle) -> JsGcStepResult { } enum BudgetedStepOutcome { - Result(JsGcStepResult), + Result(GcStepReport), Completed(BudgetedGcCycle), } @@ -4018,7 +4128,7 @@ pub(super) fn gc_drain_active_budgeted_cycle() { } } -fn gc_budgeted_step_work_units_inner(work_units: usize) -> JsGcStepResult { +fn gc_budgeted_step_work_units_inner(work_units: usize) -> GcStepReport { gc_budgeted_step_work_units_inner_with_progress(work_units, GcProgressKind::NormalIncremental) } @@ -4056,7 +4166,7 @@ pub(super) fn gc_idle_reclaim_try_start() -> bool { /// cycle stops reporting `ACTIVE`. Never starts a cycle of its own — when none /// is active the stepper's pressure check runs exactly as it would at any host /// safepoint. -pub(super) fn gc_idle_reclaim_step(budget_us: u64) -> JsGcStepResult { +pub(super) fn gc_idle_reclaim_step(budget_us: u64) -> GcStepReport { let start = Instant::now(); let mut result = gc_budgeted_step_work_units_inner(GC_NORMAL_INCREMENTAL_WORK_UNITS); while result.status == JS_GC_STEP_STATUS_ACTIVE @@ -4083,7 +4193,7 @@ fn defer_nursery_cap_to_precise_safepoint() { fn gc_budgeted_step_work_units_inner_with_progress( work_units: usize, start_progress_kind: GcProgressKind, -) -> JsGcStepResult { +) -> GcStepReport { if work_units == 0 { return gc_budgeted_status_result(); } @@ -4095,13 +4205,34 @@ fn gc_budgeted_step_work_units_inner_with_progress( return gc_budgeted_skipped_result(); }; - if !gc_budgeted_cycle_active() { + // The common host poll: no cycle, nothing due. Everything past this point + // (starting and stepping a cycle) lives out of line, so that poll does not + // pay the multi-kilobyte frame the cycle machinery needs. `_guard` stays + // held across the call, exactly as when the code was inline. + let due = if gc_budgeted_cycle_active() { + None + } else { let Some(due) = gc_budgeted_due_trigger() else { super::instruments::note_budgeted_step_skip( super::instruments::BudgetedStepSkip::NoTrigger, ); return gc_idle_step_result(); }; + Some(due) + }; + gc_budgeted_start_or_step(due, work_units, start_progress_kind) +} + +/// The part of [`gc_budgeted_step_work_units_inner_with_progress`] that starts +/// a cycle for `due` (when `Some`, i.e. no cycle was active and a trigger was +/// due) and steps the active cycle. The caller holds `BudgetedGcStepGuard`. +#[inline(never)] +fn gc_budgeted_start_or_step( + due: Option, + work_units: usize, + start_progress_kind: GcProgressKind, +) -> GcStepReport { + if let Some(due) = due { if due == BudgetedGcTrigger::YoungScavengeCap && start_progress_kind.is_budgeted() { // ★ #7909. Starting a budgeted cycle here is strictly worse than // starting nothing, and it is self-sustaining. @@ -4222,11 +4353,25 @@ fn gc_budgeted_step_work_units_inner_with_progress( fn gc_mutator_assist_step_work_units_inner_with_progress( work_units: usize, start_progress_kind: GcProgressKind, -) -> JsGcStepResult { +) -> GcStepReport { gc_budgeted_step_work_units_inner_with_progress(work_units, start_progress_kind) } +/// A host safepoint that reports what it did, debt included. For callers +/// that read the result: `js_gc_safepoint` and tests. pub(crate) fn gc_runtime_safepoint() -> JsGcStepResult { + gc_runtime_safepoint_report().with_debt() +} + +/// The runtime's own safepoint poll (regex quanta, the microtask pump, the +/// event loop). Makes exactly the decisions [`gc_runtime_safepoint`] makes and +/// skips only the debt snapshot, which none of these callers reads. +#[inline] +pub(crate) fn gc_runtime_safepoint_poll() { + let _ = gc_runtime_safepoint_report(); +} + +fn gc_runtime_safepoint_report() -> GcStepReport { let budget = gc_progress_contract().budget_for(GcProgressKind::NormalIncremental); let Some(work_units) = budget.work_units else { return gc_budgeted_status_result(); @@ -4246,14 +4391,14 @@ fn write_gc_step_result(out: *mut JsGcStepResult, result: JsGcStepResult) -> u32 #[no_mangle] pub extern "C" fn js_gc_step_work_units(work_units: u64, out: *mut JsGcStepResult) -> u32 { let work_units = usize::try_from(work_units).unwrap_or(usize::MAX); - let result = gc_budgeted_step_work_units_inner(work_units); + let result = gc_budgeted_step_work_units_inner(work_units).with_debt(); write_gc_step_result(out, result) } #[no_mangle] pub extern "C" fn js_gc_step_us(budget_us: u64, out: *mut JsGcStepResult) -> u32 { if budget_us == 0 { - let result = gc_budgeted_status_result(); + let result = gc_budgeted_status_result().with_debt(); return write_gc_step_result(out, result); } @@ -4264,12 +4409,12 @@ pub extern "C" fn js_gc_step_us(budget_us: u64, out: *mut JsGcStepResult) -> u32 { result = gc_budgeted_step_work_units_inner(1); } - write_gc_step_result(out, result) + write_gc_step_result(out, result.with_debt()) } #[no_mangle] pub extern "C" fn js_gc_step_status(out: *mut JsGcStepResult) -> u32 { - let result = gc_budgeted_status_result(); + let result = gc_budgeted_status_result().with_debt(); write_gc_step_result(out, result) } diff --git a/crates/perry-runtime/src/gc/tenuring.rs b/crates/perry-runtime/src/gc/tenuring.rs index 72255c0b78..33ba9fdfca 100644 --- a/crates/perry-runtime/src/gc/tenuring.rs +++ b/crates/perry-runtime/src/gc/tenuring.rs @@ -488,6 +488,14 @@ pub(super) fn note_surviving_object_census(moved_bytes: usize, moved_objects: us /// /// Returns without walking when the base cap is not yet half full, when a /// census (either kind) already exists, or when the nursery is empty. +/// Has the object denomination been seeded, by an allocation census or by a +/// copying minor's survivor census? Once true it stays true for the process +/// (tests reset it). +#[inline] +pub(super) fn object_census_seeded() -> bool { + OBJECT_CENSUS_SEEDED.with(Cell::get) +} + pub(super) fn maybe_seed_object_census_from_allocation(from_space_in_use_bytes: usize) { if OBJECT_CENSUS_SEEDED.with(Cell::get) { return; diff --git a/crates/perry-runtime/src/gc/tests/young_leaf_route.rs b/crates/perry-runtime/src/gc/tests/young_leaf_route.rs index 2865f520b5..e55674ae49 100644 --- a/crates/perry-runtime/src/gc/tests/young_leaf_route.rs +++ b/crates/perry-runtime/src/gc/tests/young_leaf_route.rs @@ -4,8 +4,8 @@ //! exactly once per leaf, and a measured young generation buys none. use super::super::policy::{ - gc_budgeted_due_trigger, note_young_leaf_born_old, BudgetedGcTrigger, - ScavengeNurseryCapTestGuard, GC_OLD_RECLAIM_PENDING, + gc_budgeted_due_trigger, gc_budgeted_due_trigger_eval, note_young_leaf_born_old, + BudgetedGcTrigger, DueTriggerMemo, ScavengeNurseryCapTestGuard, GC_OLD_RECLAIM_PENDING, }; use super::super::*; use super::support::*; @@ -65,3 +65,74 @@ fn young_leaf_born_old_prioritises_the_nursery_minor_until_measured() { None => clear_young_survival_for_tests(), } } + +/// `gc_check_trigger` reuses a due answer only when evaluating again would give +/// the same one. The leaf priority is the answer that would not: it consumes +/// its flag, so the next evaluation takes the ordinary path. Every other +/// answer, including a plain `OldReclaim` after the flag is gone, repeats. +#[test] +fn only_the_leaf_priority_answer_is_unrepeatable() { + let _isolation = GcTestIsolationGuard::new(); + let _pacing = crate::gc::policy::force_moving_gc_pacing(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + let _cap_due = ScavengeNurseryCapTestGuard::due_at_bytes(1); + let filler = [b'y'; 64]; + crate::string::js_string_from_bytes(filler.as_ptr(), filler.len() as u32); + let previous_survival = last_young_survival_permille(); + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true)); + clear_young_survival_for_tests(); + + note_young_leaf_born_old(); + assert_eq!( + gc_budgeted_due_trigger_eval(), + (Some(BudgetedGcTrigger::YoungScavengeCap), false) + ); + assert_eq!( + gc_budgeted_due_trigger_eval(), + (Some(BudgetedGcTrigger::OldReclaim), true) + ); + assert_eq!( + gc_budgeted_due_trigger_eval(), + (Some(BudgetedGcTrigger::OldReclaim), true) + ); + + // The same sequence through the memo `gc_check_trigger` uses: the leaf + // answer is not reused, the answer after it is. + note_young_leaf_born_old(); + let mut memo = DueTriggerMemo::new(); + assert_eq!( + memo.get(gc_budgeted_due_trigger_eval), + Some(BudgetedGcTrigger::YoungScavengeCap) + ); + assert_eq!( + memo.get(gc_budgeted_due_trigger_eval), + Some(BudgetedGcTrigger::OldReclaim) + ); + GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(false)); + assert_eq!( + memo.get(gc_budgeted_due_trigger_eval), + Some(BudgetedGcTrigger::OldReclaim), + "a repeatable answer is reused for the rest of one gc_check_trigger call" + ); + + match previous_survival { + Some(permille) => seed_young_survival_for_tests(permille), + None => clear_young_survival_for_tests(), + } +} + +/// The memo on its own: a repeatable answer is evaluated once, an +/// unrepeatable one is never reused. +#[test] +fn due_trigger_memo_reuses_only_repeatable_answers() { + let mut memo = DueTriggerMemo::new(); + assert_eq!( + memo.get(|| (Some(BudgetedGcTrigger::YoungScavengeCap), false)), + Some(BudgetedGcTrigger::YoungScavengeCap) + ); + assert_eq!(memo.get(|| (None, true)), None); + assert_eq!( + memo.get(|| panic!("a repeatable answer must not be evaluated again")), + None + ); +} diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index 264f395c30..d98b378b1c 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -739,7 +739,7 @@ pub(crate) mod stdlib_pump { // #2532/#9696 — drain every initialized extension through the // registry; stdlib deliberately has no per-extension pump arms. run_aux_pumps(); - let _ = crate::gc::gc_runtime_safepoint(); + crate::gc::gc_runtime_safepoint_poll(); } static STDLIB_HAS_ACTIVE_FN: AtomicPtr<()> = AtomicPtr::new(null_mut()); diff --git a/crates/perry-runtime/src/promise/microtasks.rs b/crates/perry-runtime/src/promise/microtasks.rs index 78ecd46c4e..df1cb42a29 100644 --- a/crates/perry-runtime/src/promise/microtasks.rs +++ b/crates/perry-runtime/src/promise/microtasks.rs @@ -302,7 +302,7 @@ fn run_microtasks(mode: MicrotaskDrainMode) -> i32 { crate::exception::js_try_end(); crate::node_submodules::diagnostics_channel_drain_uncaught(); - let _ = crate::gc::gc_runtime_safepoint(); + crate::gc::gc_runtime_safepoint_poll(); // Phase 1 of the moving-GC project (see project_gc_one_great_moving_gc): at // the OUTERMOST microtask-pump boundary the JS stack has fully unwound, so diff --git a/crates/perry-runtime/src/regex/perex_runtime.rs b/crates/perry-runtime/src/regex/perex_runtime.rs index 1f97ce41ff..99c50f2c49 100644 --- a/crates/perry-runtime/src/regex/perex_runtime.rs +++ b/crates/perry-runtime/src/regex/perex_runtime.rs @@ -56,7 +56,7 @@ impl From for EngineError { /// that requests cancellation or forces actual collection. No input/program /// view or scratch slice is live when any poll is invoked. pub(crate) fn poll() -> Result<(), EngineError> { - crate::gc::gc_runtime_safepoint(); + crate::gc::gc_runtime_safepoint_poll(); Ok(()) } diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index 533ae24984..8564dafc24 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -142,6 +142,12 @@ "verdict": "not_a_gc_pointer", "why": "#9794 allocation-site sampling: bytes remaining until the next sample. A `Cell` countdown, decremented per allocation and reset on fire — a quantity, never an address." }, + { + "file": "crates/perry-runtime/src/arena/from_space.rs", + "name": "SEALED_YOUNG_BYTES", + "verdict": "not_a_gc_pointer", + "why": "Cached young-generation occupancy for the GC due check: a `Cell` holding a validity flag, the heap generation it was measured at (a u64 epoch), and a byte count (Eden bytes outside the current block plus the active survivor space). Quantities only; it never stores a block, object or header address." + }, { "file": "crates/perry-runtime/src/array/header_gc_slots.rs", "name": "DENSE_MOVE_LAYOUT_CLASSIFIED_SLOTS", @@ -307,7 +313,7 @@ "file": "crates/perry-runtime/src/gc/census.rs", "name": "PASS1_MARKED", "verdict": "non_moving_snapshot", - "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback.", + "why": "Real GC header addresses, deliberately untraced so the diagnostic does not keep its observed objects alive. Populated only at the end of mark propagation of a synchronous full cycle; consumed at sweep entry in the same run_to_completion invocation. The intervening full-cycle phases do not relocate or run JS callbacks. The Vec is used for membership comparisons and dropped with the census before sweep. Budgeted and minor cycles skip both boundaries. Pin re-audited 2026-09-05 after #9760 touched `gc/mod.rs`: that change is `mod heap_stats;` plus a `pub(crate) use` re-export and alters no mark/sweep control flow. `heap_stats()` is reached only from `js_bun_jsc_heap_stats` (the JS-facing `bun:jsc.heapStats()`), i.e. from mutator code, never inside a cycle, and its own module contract forbids allocation or collection during its walk. The mark-complete → sweep-entry window is unchanged. Re-audited 2026-09-05 (train125) after #9769 and #9771 touched pinned files. #9769 adds one `reg_scanner!` registration to `gc/mod.rs`; #9771 adds a feature-gated `alloc_census_init()` there and a feature-gated Rust-heap dump inside `take_census`. `alloc-census` is not in the default feature set, and decisively: `census_take_if_armed_at_full_sweep_start` does `PASS1_MARKED.with(|p| p.borrow_mut().take())` BEFORE calling `take_census`, so the snapshot has already left the thread-local by the time #9771's code runs — it cannot affect the window. Neither change alters mark/sweep control flow. Re-audited 2026-09-06 after #9831 touched `gc/policy.rs`. Its hunks are (a) the tiny-parse pressure guard's pricing (`tiny_parse_pressure_headroom_bytes`, `tiny_parse_pressure_due*`, a `Cell` byte-count base) consulted from JSON.parse's mutator-side boundaries (`gc_bump_malloc_trigger`, `gc_collect_pending_suppressed_parse`, `gc_schedule_parse_boundary_collection_if_pressure`), none of which is reachable from inside a cycle, and (b) one extra `Cell` store in `note_collection_finished_arena_occupancy`, which runs from `publish_reclaim_outcome` in the Publish subphase — after `step_sweep` has already consumed the snapshot. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-05 (train126) after #9755 restructured `gc/cycle.rs`. Its hunks are all root-scan machinery (`RootScanSubphase`, `RootScanCycleState`, the mutable-scanner iteration state), which runs BEFORE mark propagation completes; `gc/mod.rs` gains only a `mod young_log;` declaration. The bracketing is unchanged — `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep` — and a synchronous full mark-sweep still moves nothing between them. Re-pinned 2026-09-05 for the #9740 hot-TLS conversion of this file: the sole change is `thread_local!` → `crate::perry_thread_local!`, a macro-name swap with identical declaration syntax and `.with()` call sites. No control flow, no phase boundary, and no storage semantics change. Re-audited 2026-09-06 (train128) after #9794's GC diagnostics touched `gc/mod.rs` and `gc/policy.rs`: both gain diagnostic module declarations and counters only — no mark/sweep control flow, and the census bracketing in `step_mark_propagation` / `step_sweep` is unchanged. Re-audited for #9794's GC diagnostics: `gc/mod.rs` gains `mod diag_sites;` / `mod survival_diag;`, a re-export, a `diag_sites::full_started(...)` call at TRIGGER time (before mark propagation begins), and exit-time reporting. Nothing executes between mark-complete and sweep-entry, so the window is unchanged. Re-audited 2026-09-06 for the retained array-growth verifier fix: the cycle.rs change passes the existing non-copying evacuation verifier an explicit all-forwarded policy. That call remains in minor finalization, outside the synchronous full-cycle census window; its root and heap reads do not allocate GC objects, move objects, or invoke JS callbacks. The mark-complete and sweep-entry boundaries are unchanged. Re-audited 2026-09-05 after #9830 touched `gc/policy.rs`. That change is (a) six `thread_local! {` blocks rewritten as `crate::perry_thread_local! {` and (b) one `#[cfg(test)]` accessor listing the trigger path's hot-slot indices. The macro keeps the same storage, the same `.with()` at every read and write, and the same destructor registration (the teardown guard exists exactly when `needs_drop` holds, which is what `std::thread_local!` already decided); no value, predicate or branch in the file changes, so no mark or sweep control flow does. The one new behaviour is on a declaration's FIRST read: `HotKey::resolve_and_cache` takes a mutex and allocates a key through the GLOBAL allocator. Even if a first read landed inside this window it would be sound — the window's contract is that nothing relocates and no JS callback runs, and a mimalloc allocation does neither. `census_pass1_if_armed` is still inside `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` inside `step_sweep`; the bracketing is untouched. Re-audited 2026-09-06 (train132) after #9860 and #9845 touched `gc/mod.rs`. Both hunks are re-export lists and nothing else: #9860 adds `idle_reclaim_elapsed_starts` / `IDLE_RECLAIM_REARM_MS`, and #9845 adds `owner_is_dead_copied_minor_from_space_of_type`. No mark or sweep control flow changes. #9845's substantive work sits in `gc/oldgen.rs` and `gc/copying.rs`, neither pinned: the copying-minor arm (`finalize_dead_copied_minor_from_space_regexps`) runs on a MINOR, which skips both census boundaries; the full-cycle arm (`collect_dead_registered_regexps_post_trace`, from `with_dead_collection_finalize`) walks the RegExp registry building a Vec of addresses — no GC allocation, no JS callback, so it cannot relocate the snapshot's subjects — and it is reached from the sweep body, i.e. AFTER `census_take_if_armed_at_full_sweep_start` has already `take()`n the snapshot out of the thread-local. The mark-complete -> sweep-entry window is unchanged. Re-audited 2026-09-07 for #9965 after 1ec9e0e8a touched `gc/cycle.rs` and `gc/mod.rs`: `gc/mod.rs:216-217` only declares and imports the failure-attribution module, while `gc/cycle.rs:1414-1417` reads the trigger and diagnostic counters immediately before evacuation verification inside `atomic_finalize_minor_prelude`. Full cycles bypass `MinorPrelude` at `gc/cycle.rs:1192-1196`; evacuation remains guarded by the minor-only context at `gc/cycle.rs:1330-1372`. The snapshot store remains at `gc/cycle.rs:963-964` after synchronous full marking, and its take remains at `gc/cycle.rs:1454-1457` before sweep. No new write, relocation, collection, or JS callback was added to that full-cycle interval, so the PASS1_MARKED window is unaffected. Re-audited 2026-09-07 for the regex census rows: all new work is in `take_census` after `census_take_if_armed_at_full_sweep_start` has taken PASS1_MARKED out of TLS; neither boundary nor the intervening cycle control flow changed. Re-audited 2026-09-08 (train144) after #9976 and #9977 touched pinned files. `gc/mod.rs` gains exactly three lines: `mod copying_phase;` and `mod regex_census;` (declarations) and one `reg_scanner!(regex::site_test::scan_roots_mut)` registration. A scanner registration adds a root SOURCE for the mutable-root walks; it does not move either census boundary and runs nowhere between them. `gc/census.rs` widens `side_tables()` to `pub(super)`, extends it with regex rows and adds a test module — all census REPORTING, which runs from the diagnostic dump, not inside a cycle. Mark/sweep control flow between `census_pass1_if_armed` and `census_take_if_armed_at_full_sweep_start` is untouched. Re-audited 2026-09-08 for #9849 JSON construction deferral. `gc/mod.rs` adds the `json_defer` module/re-export and a trusted-header layout helper used only by already-validated JSON emitters; neither changes or runs in collector phase control flow. `gc/policy.rs` adds JSON completion scheduling, construction-grace checks, and safepoint deferral predicates. These are called from mutator-side JSON allocation/output boundaries and ordinary safepoint entry; they do not alter `step_mark_propagation`, `step_sweep`, or invoke callbacks or relocation between the census boundaries. The mark-complete to sweep-entry window is unchanged. The follow-up adds a cfg(test)-only one-shot boolean for deterministic explicit-pressure fixtures; it is absent from production builds and cannot affect the census window. The first predicate read consumes it, so post-parse accounting exercises normal pricing. Re-audited 2026-09-09 for bounded tiny-JSON completion polling. The policy.rs changes split the mutator-side pending-parse check into an inlined empty fast path plus an outlined debt-service path, and amortize the mutator-side arena-pressure read across 64 bounded parse completions. Neither function is reachable from step_mark_propagation or step_sweep; neither census boundary nor the synchronous full-cycle interval between them changes. Re-audited 2026-09-09 for lazy JSON record batches: policy.rs only widens gc_budgeted_cycle_active visibility from pub(super) to pub(crate). Its body remains a read-only Cell query. The new caller is lazy_get materialization in the mutator; run_to_completion, step_mark_propagation, census snapshot consumption at step_sweep, and the synchronous non-moving window are unchanged. Re-audited 2026-09-09 for completed JSON-output debt: the added gc_service_json_output_sweep function calls the existing trigger check from a rooted mutator boundary and reports whether its malloc-count request remains due. It is not called from any census or collector phase; the synchronous mark-complete to sweep-entry window is unchanged. Re-audited 2026-09-09 for the JSON byte-debt carry: the same mutator-only service helper now distinguishes requests satisfied before its call from those satisfied by its trigger check. The added enum contains no payload, both count reads are scalar, and no census boundary or collector phase changed. Re-audited 2026-09-11 for #10055: gc/mod.rs only registers the weak UTF-16 index scanner during gc_init. It neither marks strings nor allocates GC objects or runs JS; offset vectors use the Rust allocator. The mark-complete to sweep-entry census window and cycle control flow are unchanged. Re-audited 2026-09-11 for #10054: gc/mod.rs adds only the trim-cache mutable-root scanner registration in gc_init. Its scanner visits two existing string slots without allocating or invoking JS. Root scanning still precedes mark completion, and neither census boundary nor the synchronous mark-complete to sweep-entry window changes. Re-audited 2026-09-11 for #10060: the census array classifier now reads the logical element start and bounds its scan by the remaining capacity. The helper only reads the existing GC/header words and performs pointer arithmetic; it cannot allocate, collect, or call JS. This classifier runs in take_census after PASS1_MARKED has been taken out of TLS. Neither census boundary nor the mark-complete to sweep-entry control flow changed. Re-audited for #8512: gc/mod.rs only enables the existing PTY mutable-root scanner on Windows; it changes no mark/sweep phase or census boundary. The scanner visits NaN-boxed slots without running JS callbacks. Re-audited 2026-09-12 for the single regular-expression engine: `gc/mod.rs` changes `mod prefetch;` to `pub(crate) mod prefetch;` so the RegExp owner-table walks can prefetch headers, a visibility change with no new call in collector control flow; `gc/census.rs` changes only its `#[cfg(test)]` `regex_census_tests` module, dropping assertions for the previous engine's cache rows. Neither boundary (`census_pass1_if_armed` in `step_mark_propagation`, `census_take_if_armed_at_full_sweep_start` in `step_sweep`) nor the synchronous mark-complete to sweep-entry interval changes. Re-audited 2026-09-13 after the #10169 fix touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` gains only `pub(crate) use` re-exports (`policy::note_young_leaf_born_old`, `policy::young_generation_holds_a_nursery`, `promote_in_place::{young_generation_measured_dying, young_generation_measured_retained}`, and cfg(test) survival seeders). `gc/policy.rs` gains a `Cell` thread-local (`GC_YOUNG_LEAF_BORN_OLD`, no pointer), its setter, a pure predicate over `copying_from_space_in_use_bytes` vs the base nursery cap, and a consumed-once branch at the top of `gc_budgeted_due_trigger` that may answer `YoungScavengeCap` ahead of `OldReclaim`. That branch decides WHICH collection a safepoint starts (a minor instead of a full); it runs before any cycle begins and never inside one, so the mark-complete → sweep-entry window of a synchronous full — where PASS1_MARKED is populated and consumed within one `run_to_completion` — is unchanged, and neither hunk adds an allocation, a JS callback, or a relocation to it. Re-audited 2026-09-13 for the heap generation (#10164 cross-call search positions): `gc/mod.rs` only declares `pub(crate) mod heap_generation;`. `gc/cycle.rs` wraps the `Sweep` and `Reclaim` arms of `GcCycleState::step` in a `HeapChange` scope and opens one inside `atomic_finalize_minor_prelude`'s evacuation branch (with a nested one around old-page defrag). Opening and closing a scope only increments two thread-local integer cells (`HEAP_GENERATION`, `OPEN_HEAP_CHANGES`); a first thread-local read may allocate a key through the global allocator, which neither relocates nor runs JS. The `Sweep` scope opens immediately before `step_sweep`, i.e. before `census_take_if_armed_at_full_sweep_start` takes PASS1_MARKED out of TLS, and adds no relocation, collection or JS callback to the synchronous mark-complete to sweep-entry window; the minor-prelude scope is unreachable from a full cycle, which bypasses `MinorPrelude`. Neither boundary nor the intervening control flow changed. Re-audited 2026-09-13 for #10182 block-granular reclamation, which touched `gc/cycle.rs`. Two hunks: (a) in the `RememberedSetRebuild` subphase of AtomicFinalize — INSIDE the window — the require-marked old-to-young rebuild is now constructed with `OldToYoungRememberedRebuildState::new_skipping`, whose cursor never enters blocks the census recorded as holding no reached, pinned or pre-marked object (`BlockCensus::unmarked_blocks`); computing that list reads `arena_block_snapshots()` and allocates one `Vec` through the global allocator. It visits a subset of the same objects the rebuild already walked (every skipped object would have been rejected as unmarked), and it neither allocates a GC object, relocates anything, nor runs a JS callback. (b) In `step_sweep`, `IncrementalSweepState::with_block_skip` runs after `census_take_if_armed_at_full_sweep_start` has already taken PASS1_MARKED out of TLS. Neither boundary moved and the synchronous mark-complete to sweep-entry interval gains no relocation, collection or callback. Re-audited 2026-09-14 for the GC due-check fast path, which touched `gc/mod.rs` and `gc/policy.rs`. `gc/mod.rs` only changes the safepoint re-exports: `gc_runtime_safepoint` becomes cfg(test) and `gc_runtime_safepoint_poll` is added. `gc/policy.rs`: the budgeted step returns a debt-free `GcStepReport` (debt is attached by the FFI and test entry points after the step returns) and moves cycle start/step into an out-of-line `gc_budgeted_start_or_step`; `gc_check_trigger` reuses a repeatable due-trigger answer through `DueTriggerMemo`, placed after its `GC_FLAG_IN_ALLOC` and suppression early returns; the young scavenge cap reuses the old-gen pressure value the due trigger already read and checks the census-seeded flag first. All of it runs from mutator safepoints, allocation-point trigger checks and the host step API, before a cycle starts or between budgeted steps. None of it is reachable between `census_pass1_if_armed` in `step_mark_propagation` and `census_take_if_armed_at_full_sweep_start` in `step_sweep` of a synchronous full: an allocation inside that window reaches `gc_check_trigger` with `GC_FLAG_IN_ALLOC` set and returns before the changed code. No allocation, relocation, collection or JS callback is added to the window.", "window": { "start": { "file": "crates/perry-runtime/src/gc/census.rs", @@ -324,8 +330,8 @@ "sources": { "crates/perry-runtime/src/gc/census.rs": "3f9e6be47b4454022b70ff2357bbdf3a80ef6a84c4986e58763acbdcce9142c1", "crates/perry-runtime/src/gc/cycle.rs": "fdef083301463ad8cec8142ca86cc4354e289c67e97bbc6caea2e8d16d4bf089", - "crates/perry-runtime/src/gc/mod.rs": "90339683735e4d662628cd98279a1972d523c3f2ebc358130ed3bdb0c6fc2d3f", - "crates/perry-runtime/src/gc/policy.rs": "aea89274a017156efea3516b4f49124f48a66527a08195b662cfbf61672ac042", + "crates/perry-runtime/src/gc/mod.rs": "fa188476e306e918dd0a967e49e8f204edccac6b8cf8dd57aec69b38235e3afa", + "crates/perry-runtime/src/gc/policy.rs": "59ae22b9e87afb0ddcdee582f867c05d2224dbafa93ad3ce70cb109e34bd6237", "crates/perry-runtime/src/gc/progress.rs": "a5ad3971bbe4047229ca57325234780daa85921dbc778e1c08dff4ad07ccfb96" } } From 4ba102d0ea1e59a95a4deec31975d12b9c53e879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Mon, 14 Sep 2026 06:51:22 +0200 Subject: [PATCH 2/2] changelog: key the GC due-check fragment to PR 10253 --- changelog.d/10253-gc-due-check-fast-path.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog.d/10253-gc-due-check-fast-path.md diff --git a/changelog.d/10253-gc-due-check-fast-path.md b/changelog.d/10253-gc-due-check-fast-path.md new file mode 100644 index 0000000000..8efb471ea1 --- /dev/null +++ b/changelog.d/10253-gc-due-check-fast-path.md @@ -0,0 +1,13 @@ +Made the GC's "is a collection due?" check cheap when nothing is due, without +changing any collection decision. Runtime safepoint polls (regex search quanta, +the microtask pump, the event loop) no longer build a step result whose debt +snapshot nobody read. The young-generation occupancy the nursery cap compares +against is now O(1): the bytes outside Eden's current block are cached per heap +generation, and every move of an arena's current block goes through +`Arena::set_current`, which invalidates the cache. Debug builds check every cached +answer against the block walk. `gc_check_trigger`, reached from every `gc_malloc` +and JSON parse, evaluates the due trigger once instead of up to three times. +Measured at the shipping release profile: 1M hoisted `re.exec` −12.5 % +instructions, 400k small `JSON.parse` −12.3 %, 1M hoisted `re.test` −6.6 %, +with async, allocation, string, BigInt and Map workloads unchanged and peak RSS +unchanged.