perf(regex): resume JS-level searches on non-ASCII strings from the previous call's position

changelog.d/10205-regex-cross-call-position.md

@@ -0,0 +1,3 @@
+### Performance
+
+- **JavaScript regex loops over a non-ASCII string resume where the previous call stopped** (#10205). A `while (re.exec(s))` or `for (const m of s.matchAll(re))` loop used to pay a seek from an end of the string on every call, so a loop over one long string with umlauts, CJK or emoji did quadratic work. For 80,000 matches the `exec` loop drops from 28.5 s to 314 ms, and `matchAll` from 26.9 s to 463 ms. A new per-thread heap generation, advanced by every collection that frees or moves memory, is what lets a remembered position be matched to the same string safely; RegExp objects and strings gain no state.

crates/perry-runtime/src/arena/promote.rs

@@ -303,6 +303,9 @@ pub(crate) fn finish_in_place_promotion(
     promotion: InPlacePromotion,
     liveness: PromotionLiveness,
 ) -> InPlacePromotionStats {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Promotion,
+    );
     let mut stats = InPlacePromotionStats::default();
     if promotion.blocks.is_empty() {
         return stats;
@@ -567,6 +570,7 @@ fn stamp_and_index_block(block: &ArenaBlock, liveness: PromotionLiveness) -> (us
 /// `InPlacePromotion::reserved_bytes`), so the mutator maps only what it
 /// actually allocates.
 fn reset_young_after_promotion() {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     crate::gc::ARENA_FREE_LIST.with(|fl| fl.borrow_mut().clear());
     crate::gc::ARENA_FREE_LIST_NONEMPTY.with(|c| c.set(false));
 

crates/perry-runtime/src/arena/reset.rs

@@ -11,6 +11,9 @@ use super::*;
 /// nothing escapes, GC observes that all 700k+ objects from the
 /// previous burst are dead and reclaims the entire arena in O(1).
 pub fn arena_reset_all_blocks_to_zero() {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     // Only the general arena is reset (issue #179). The longlived arena
     // holds cached data that must not be reclaimed.
     ARENA.with(|arena| unsafe {
@@ -85,6 +88,7 @@ fn poison_region_in_place(arena: &mut Arena) {
 }
 
 fn reset_region_to_zero(arena: &mut Arena) -> (usize, usize) {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     let mut reset_blocks = 0usize;
     let mut reusable_bytes = 0usize;
     for block in arena.blocks.iter_mut() {
@@ -171,6 +175,7 @@ pub(crate) fn active_survivor_block_index_range() -> std::ops::Range<usize> {
 /// has. No other reclaim path (the non-moving minor's `arena_reset_empty_blocks`,
 /// the full mark-sweep, old-gen defrag) is affected by that knob.
 pub(crate) fn copying_reset_from_spaces_and_flip() -> ArenaResetStats {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     if protect_fromspace_enabled() {
         return copying_quarantine_from_spaces_and_flip();
     }
@@ -261,6 +266,7 @@ pub(crate) fn copying_reset_from_spaces_and_flip() -> ArenaResetStats {
 /// in place and the inline allocator keeps reusing the same ~8MB
 /// arena block forever.
 pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     let n_live = block_has_live.iter().filter(|&&b| b).count();
     let n_total = block_has_live.len();
     // Issue #179: only reset general-arena blocks. Longlived-arena blocks
@@ -508,6 +514,7 @@ pub fn arena_reset_empty_blocks(block_has_live: &[bool]) -> ArenaResetStats {
 const GENERAL_DEALLOC_DEAD_CYCLES: u32 = 2;
 
 fn filter_free_list_ranges(ranges: &[(usize, usize)]) {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     if ranges.is_empty() {
         return;
     }
@@ -639,6 +646,7 @@ impl ArenaResetEmptyBlocksState {
     }
 
     fn process_reset_block(&mut self, block_idx: usize) -> Option<(usize, usize, usize)> {
+        crate::gc::heap_generation::debug_assert_heap_change_open();
         let snapshot = self.snapshots.get(block_idx).copied().unwrap_or_default();
         if snapshot.data == 0 {
             return None;
@@ -695,6 +703,7 @@ impl ArenaResetEmptyBlocksState {
         &mut self,
         block_idx: usize,
     ) -> Result<(usize, usize, ArenaBlockRelease), DeallocReject> {
+        crate::gc::heap_generation::debug_assert_heap_change_open();
         let snapshot = self.snapshots.get(block_idx).copied().unwrap_or_default();
         if snapshot.data == 0 {
             return Err(DeallocReject::NoSnapshot);
@@ -882,6 +891,7 @@ impl SurvivorArenaReclaimState {
     }
 
     fn process_block(&mut self, local_idx: usize) {
+        crate::gc::heap_generation::debug_assert_heap_change_open();
         let global_idx = self.block_start + local_idx;
         let snapshot = self.snapshots.get(global_idx).copied().unwrap_or_default();
         if snapshot.data == 0 {
@@ -1184,6 +1194,7 @@ impl OldArenaReclaimDeadBlocksState {
     }
 
     fn process_block(&mut self, local_idx: usize) {
+        crate::gc::heap_generation::debug_assert_heap_change_open();
         let old_block_start = longlived_end();
         let block_idx = old_block_start + local_idx;
         if self
@@ -1298,6 +1309,7 @@ impl OldArenaReclaimDeadBlocksState {
 }
 
 pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaResetStats {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     let old_block_start = longlived_end();
     let stats = OLD_ARENA.with(|arena| unsafe {
         let arena = &mut *arena.get();
@@ -1392,6 +1404,7 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks(
     block_has_live: &[bool],
     selected_old_blocks: &crate::fast_hash::PtrHashSet<usize>,
 ) -> ArenaResetStats {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     if selected_old_blocks.is_empty() {
         return ArenaResetStats::default();
     }
@@ -1492,6 +1505,7 @@ fn reclaim_dead_survivor_arena_blocks(
     block_start: usize,
     block_has_live: &[bool],
 ) -> ArenaResetStats {
+    crate::gc::heap_generation::debug_assert_heap_change_open();
     with_survivor_arena_mut(arena_idx, |arena| {
         let keep_idx = arena
             .blocks

crates/perry-runtime/src/arena/tests.rs

@@ -31,6 +31,9 @@ pub(super) fn run_with_fresh_arenas(test: impl FnOnce() + Send + 'static) {
 }
 
 fn reset_old_nursery_block(dead_cycles_before: u32) -> (usize, usize, usize, ArenaResetStats) {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     let mut blocks = Vec::new();
     for _ in 0..7 {
         let ptr = arena_alloc(BLOCK_SIZE, 8) as usize;
@@ -69,6 +72,9 @@ fn reset_old_nursery_block(dead_cycles_before: u32) -> (usize, usize, usize, Are
 fn reset_single_reclaimable_nursery_block(
     dead_cycles_before: u32,
 ) -> (usize, usize, usize, usize, ArenaResetStats) {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     let mut blocks = Vec::new();
     for _ in 0..6 {
         let ptr = arena_alloc(BLOCK_SIZE, 8) as usize;
@@ -144,6 +150,9 @@ fn object_start_bitmap_stamps_only_maps_and_clears_on_reset() {
 #[test]
 fn survivor_reclaim_resets_dead_blocks() {
     run_with_fresh_arenas(|| {
+        let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+            crate::gc::heap_generation::HeapChangeKind::Sweep,
+        );
         let baseline = arena_telemetry_snapshot();
         let _dead = arena_alloc_gc_survivor(2 * 1024 * 1024, 8, GC_TYPE_STRING);
         let after_alloc = arena_telemetry_snapshot();
@@ -177,6 +186,9 @@ fn survivor_reclaim_resets_dead_blocks() {
 #[test]
 fn budgeted_survivor_reclaim_accumulates_release_stats_across_slices() {
     run_with_fresh_arenas(|| {
+        let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+            crate::gc::heap_generation::HeapChangeKind::Sweep,
+        );
         for _ in 0..3 {
             let ptr = arena_alloc_gc_survivor(BLOCK_SIZE, 8, GC_TYPE_STRING);
             assert!(!ptr.is_null());
@@ -838,6 +850,9 @@ fn longlived_pointer_is_disjoint_from_general_blocks() {
 
 #[test]
 fn test_arena_reset_reuses_dead_general_block_without_touching_live_block() {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     let mut dead_blocks = Vec::new();
 
     for _ in 0..6 {
@@ -954,6 +969,9 @@ fn longlived_walk_yields_indices_outside_general_range() {
 /// is the only thing referencing them.
 #[test]
 fn reset_never_clears_longlived_blocks() {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     let ll = arena_alloc_gc_longlived(40, 8, GC_TYPE_STRING) as usize;
     let ll_header_in_block = {
         // The header sits GC_HEADER_SIZE before the user pointer;
@@ -1061,6 +1079,9 @@ fn old_gen_walk_yields_indices_after_longlived() {
 /// block is marked dead. Promotion implies indefinite lifetime.
 #[test]
 fn reset_never_clears_old_gen_blocks() {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::Sweep,
+    );
     let old_ptr = arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize;
     let old_header = old_ptr - GC_HEADER_SIZE;
     let n_blocks = arena_block_count();

crates/perry-runtime/src/gc/copying.rs

@@ -500,6 +500,7 @@ impl CopyingNurseryCollector {
     }
 
     pub(super) unsafe fn move_young(&mut self, ptr: CopyingPointer) -> usize {
+        crate::gc::heap_generation::debug_assert_heap_change_open();
         let header = ptr.header;
         let old_user = (header as *mut u8).add(GC_HEADER_SIZE);
         let flags = (*header).gc_flags;
@@ -1098,6 +1099,9 @@ pub(super) fn run_copied_minor_attempt(
     _trigger_kind: GcTriggerKind,
     may_speculate: bool,
 ) -> CopiedMinorAttempt {
+    let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+        crate::gc::heap_generation::HeapChangeKind::CopyingMinor,
+    );
     if let Some(trace) = trace.as_mut() {
         trace.copying_nursery = eligibility.trace_stats();
         trace.legacy_copy_only_scanner_pinned = eligibility.legacy_root_stats;

crates/perry-runtime/src/gc/cycle.rs

@@ -772,8 +772,18 @@ impl GcCycleState {
             GcCyclePhase::MarkPropagation => self.step_mark_propagation(budget),
             GcCyclePhase::BlockPersistence => self.step_block_persistence(budget),
             GcCyclePhase::AtomicFinalize => self.step_atomic_finalize(budget),
-            GcCyclePhase::Sweep => self.step_sweep(budget),
-            GcCyclePhase::Reclaim => self.step_reclaim(budget),
+            GcCyclePhase::Sweep => {
+                let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+                    crate::gc::heap_generation::HeapChangeKind::Sweep,
+                );
+                self.step_sweep(budget)
+            }
+            GcCyclePhase::Reclaim => {
+                let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+                    crate::gc::heap_generation::HeapChangeKind::Reclaim,
+                );
+                self.step_reclaim(budget)
+            }
             GcCyclePhase::Complete => {}
         }
         self.active_step_start = None;
@@ -1370,6 +1380,9 @@ impl GcCycleState {
         let mut evacuation = EvacuationTraceStats::default();
         let mut evacuation_sticky = StickyRememberedSet::default();
         if minor.evacuation_policy.enabled {
+            let _heap_change = crate::gc::heap_generation::HeapChange::begin(
+                crate::gc::heap_generation::HeapChangeKind::Evacuation,
+            );
             let phase_start = trace_phase_start(&self.trace);
             let mut evacuated_new_headers = Vec::new();
             let mut evacuated_original_headers = Vec::new();
@@ -1378,11 +1391,16 @@ impl GcCycleState {
                 &mut evacuated_new_headers,
                 &mut evacuated_original_headers,
             );
-            let old_page_evacuation = evacuate_selected_old_pages_collecting(
-                &minor.old_page_selection.pages,
-                &mut evacuated_new_headers,
-                &mut evacuated_original_headers,
-            );
+            let old_page_evacuation = {
+                let _compaction = crate::gc::heap_generation::HeapChange::begin(
+                    crate::gc::heap_generation::HeapChangeKind::Compaction,
+                );
+                evacuate_selected_old_pages_collecting(
+                    &minor.old_page_selection.pages,
+                    &mut evacuated_new_headers,
+                    &mut evacuated_original_headers,
+                )
+            };
             evacuation.objects = evacuation
                 .objects
                 .saturating_add(old_page_evacuation.objects);

crates/perry-runtime/src/gc/heap_generation.rs

@@ -0,0 +1,135 @@
+//! A per-thread heap generation that advances whenever heap memory is freed or
+//! moved.
+//!
+//! An address observed while the generation reads `G` still names the same
+//! object for as long as the generation still reads `G`: freeing that object
+//! (so its address can be handed to a new allocation) or relocating it both
+//! advance the generation first. RegExp's cross-call search position (#10164)
+//! uses this to recognise that a string it searched on a previous call is the
+//! same string, without adding a traced edge or any per-object state.
+//!
+//! # The funnel
+//!
+//! Every free or move of heap memory runs inside a [`HeapChange`] scope, which
+//! advances the generation when it opens and again when it closes. Opening
+//! covers observers that recorded an address before the event; closing covers
+//! any observer that recorded one while the event was running (a JS callback
+//! reached from inside a collection). Scopes may nest.
+//!
+//! The primitives that make an object's memory reusable or give it a new
+//! address call [`debug_assert_heap_change_open`]: the arena region and block
+//! resets, the old-generation free-list rebuild, dead-object reclaim, the
+//! malloc sweep, promotion's young reset, evacuation (copying minor, tenured
+//! nursery, selected old pages), forwarding-stub release and `gc_realloc`.
+//! A free or move reached outside every scope panics in debug builds, so a new
+//! path cannot silently bypass the generation.
+//!
+//! Recycling a block that is already empty (the block pool, the from-space
+//! quarantine ring, an arena dropped at thread exit) needs no scope: the
+//! objects that lived there were freed or moved by an event that already
+//! advanced the generation, and nothing has been allocated there since.
+
+use std::cell::Cell;
+
+crate::perry_thread_local! {
+    static HEAP_GENERATION: Cell<u64> = const { Cell::new(0) };
+    static OPEN_HEAP_CHANGES: Cell<u32> = const { Cell::new(0) };
+}
+
+/// What kind of event a [`HeapChange`] scope covers.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[repr(u8)]
+pub(crate) enum HeapChangeKind {
+    /// A copying (evacuating) minor: from-space reset, young moves, promotion.
+    CopyingMinor = 0,
+    /// A non-moving or full sweep step: arena resets, malloc frees, dead-object
+    /// reclaim, free-list rebuild.
+    Sweep = 1,
+    /// An incremental reclaim step of a budgeted cycle.
+    Reclaim = 2,
+    /// Minor-prelude evacuation of tenured nursery objects and forwarding-stub
+    /// release.
+    Evacuation = 3,
+    /// Old-generation compaction or defragmentation.
+    Compaction = 4,
+    /// Promotion of the young generation outside a copying minor.
+    Promotion = 5,
+    /// A malloc-tracked object reallocated to a new address.
+    Realloc = 6,
+}
+
+#[cfg(test)]
+const HEAP_CHANGE_KINDS: usize = 7;
+
+#[cfg(test)]
+crate::perry_thread_local! {
+    static HEAP_CHANGES_BY_KIND: Cell<[u64; HEAP_CHANGE_KINDS]> =
+        const { Cell::new([0; HEAP_CHANGE_KINDS]) };
+}
+
+/// This thread's current heap generation.
+#[inline]
+#[cfg_attr(not(feature = "regex-engine"), allow(dead_code))]
+pub(crate) fn heap_generation() -> u64 {
+    HEAP_GENERATION.with(Cell::get)
+}
+
+#[inline]
+fn advance() {
+    // `try_with`: a scope can close while thread-locals are being destroyed.
+    let _ = HEAP_GENERATION.try_with(|g| g.set(g.get().wrapping_add(1)));
+}
+
+/// A region of code that may free or move heap memory. See the module docs.
+#[must_use = "a HeapChange covers only the code that runs while it is held"]
+pub(crate) struct HeapChange {
+    _not_send: std::marker::PhantomData<*const ()>,
+}
+
+impl HeapChange {
+    #[inline]
+    pub(crate) fn begin(kind: HeapChangeKind) -> Self {
+        advance();
+        let _ = OPEN_HEAP_CHANGES.try_with(|n| n.set(n.get() + 1));
+        #[cfg(test)]
+        let _ = HEAP_CHANGES_BY_KIND.try_with(|c| {
+            let mut counts = c.get();
+            counts[kind as usize] += 1;
+            c.set(counts);
+        });
+        #[cfg(not(test))]
+        let _ = kind;
+        Self {
+            _not_send: std::marker::PhantomData,
+        }
+    }
+}
+
+impl Drop for HeapChange {
+    #[inline]
+    fn drop(&mut self) {
+        let _ = OPEN_HEAP_CHANGES.try_with(|n| n.set(n.get().saturating_sub(1)));
+        advance();
+    }
+}
+
+/// Called by every primitive that frees or moves heap memory.
+#[inline]
+#[track_caller]
+pub(crate) fn debug_assert_heap_change_open() {
+    #[cfg(debug_assertions)]
+    {
+        let open = OPEN_HEAP_CHANGES.try_with(Cell::get).unwrap_or(1);
+        assert!(
+            open > 0,
+            "heap memory freed or moved outside a HeapChange scope; the heap generation \
+             would not advance and an address-keyed observer could confuse two objects"
+        );
+    }
+}
+
+/// How many scopes of `kind` have opened on this thread.
+#[cfg(test)]
+pub(crate) fn heap_changes_of_kind(kind: HeapChangeKind) -> u64 {
+    HEAP_CHANGES_BY_KIND.with(|c| c.get()[kind as usize])
+}