perf(regex): skip the unobservable exec lookup, keep small match scratch inline, copy ASCII captures in one pass (#10166)

changelog.d/10212-regex-per-call.md

@@ -0,0 +1,6 @@
+Made each `RegExp.prototype.test` and `exec` call cheaper. When a RegExp, its
+prototype and `exec` are the untouched builtins, the unobservable `exec`
+property lookup is skipped. Match scratch for programs of up to 32 registers
+and 16 captures is held inline instead of heap-allocated per call, and
+captures of an ASCII string are copied as one byte range instead of being
+decoded and re-encoded twice.

crates/perry-runtime/src/gc/tests/runtime_roots/perex_dispatch.rs

@@ -460,3 +460,73 @@ fn perex_dispatch_proxy_apply_getter_and_nested_trap_survive_movement() {
         );
     }
 }
+
+#[test]
+fn perex_dispatch_skips_the_exec_lookup_only_when_nothing_can_observe_it() {
+    // The guard holds the global side-table lock, which the prototype edits
+    // below need; taking it again here would deadlock.
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    super::perex_public::register_host_roots();
+    let scope = RuntimeHandleScope::new();
+    let miss = text(&scope, b"x");
+    let hit = text(&scope, b"NEVER");
+    let lookups = || dispatch::EXEC_LOOKUPS.with(Cell::get);
+    let yes = function(&scope, return_this as *const u8, 1);
+
+    // Untouched: the builtin runs with no lookup, and still answers. The
+    // realm records RegExp.prototype's canonical site on first use, so the
+    // first call on a thread may take the lookup; none after it does.
+    let plain = regex(&scope);
+    assert!(!test(&plain, &miss));
+    let before = lookups();
+    for _ in 0..10 {
+        assert!(!test(&plain, &miss));
+        assert!(test(&plain, &hit));
+    }
+    assert_eq!(
+        lookups(),
+        before,
+        "an untouched RegExp needs no exec lookup"
+    );
+
+    // An own exec is found by the lookup, and runs.
+    let own = regex(&scope);
+    put(&own, b"exec", &yes);
+    let before = lookups();
+    assert!(test(&own, &miss), "an own exec override must run");
+    assert!(lookups() > before);
+
+    // A reparented RegExp resolves exec on its new prototype.
+    let reparented = regex(&scope);
+    let parent = object(&scope);
+    put(&parent, b"exec", &yes);
+    assert_eq!(
+        crate::proxy::js_reflect_set_prototype_of(
+            reparented.get_nanbox_f64(),
+            parent.get_nanbox_f64()
+        )
+        .to_bits(),
+        crate::value::TAG_TRUE
+    );
+    let before = lookups();
+    assert!(
+        test(&reparented, &miss),
+        "the new prototype's exec must run"
+    );
+    assert!(lookups() > before);
+
+    // Replacing RegExp.prototype.exec reaches every RegExp, including a fresh
+    // one; restoring it restores the skipped lookup.
+    let proto = scope.root_nanbox_f64(crate::object::builtin_prototype_value("RegExp"));
+    let original = scope.root_nanbox_f64(api::finish(dispatch::get(&proto, b"exec")));
+    put(&proto, b"exec", &yes);
+    let fresh = regex(&scope);
+    let before = lookups();
+    assert!(test(&fresh, &miss), "a replaced prototype exec must run");
+    assert!(lookups() > before);
+    put(&proto, b"exec", &original);
+    let before = lookups();
+    assert!(!test(&fresh, &miss));
+    assert_eq!(lookups(), before);
+}

crates/perry-runtime/src/gc/tests/runtime_roots/perex_public.rs

@@ -348,3 +348,59 @@ fn perex_public_nonglobal_test_propagates_lastindex_coercion_throw() {
     assert_eq!(RuntimeHandleScope::active_len_for_tests(), roots);
     assert_ne!(address::<StringHeader>(&input), before);
 }
+
+#[test]
+fn perex_public_exec_captures_agree_across_inline_and_heap_slots_and_storage() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _scan = ConservativeScanDisabledGuard::new();
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    let _force = ForcedEvacuationTestGuard::on();
+    register_host_roots();
+    let twenty_groups = "(a)".repeat(20);
+    let twenty_a = "a".repeat(20);
+    // A few captures fit the inline slots. Twenty groups need 42 registers and
+    // 21 capture spans, past both inline limits. The alternation backtracks,
+    // growing frames through a rebuffer. `None` is an unset group.
+    let cases: [(&str, &str, Vec<Option<&str>>); 4] = [
+        ("(a)(b)?c", "acz", vec![Some("ac"), Some("a"), None]),
+        (&twenty_groups, &twenty_a, {
+            let mut all = vec![Some(twenty_a.as_str())];
+            all.extend(std::iter::repeat_n(Some("a"), 20));
+            all
+        }),
+        (
+            "(a|ab)(c|bcd)(d*)",
+            "abcd",
+            vec![Some("abcd"), Some("a"), Some("bcd"), Some("")],
+        ),
+        ("(x*)$", "abc", vec![Some(""), Some("")]),
+    ];
+    for (pattern, tail, expected) in cases {
+        // The same match behind an ASCII prefix and a non-ASCII one, so both
+        // the byte copy and the unit-by-unit copy produce these captures.
+        for prefix in ["!", "\u{e9}"] {
+            let scope = RuntimeHandleScope::new();
+            let receiver = regex(&scope, pattern, "");
+            let subject = format!("{prefix}{tail}");
+            let input = text(&scope, subject.as_bytes());
+            let result = exec(&receiver, &input);
+            assert!(!result.is_null(), "/{pattern}/ must match {subject:?}");
+            let result = scope.root_raw_mut_ptr(result);
+            for (index, want) in expected.iter().enumerate() {
+                let value = item(&result, index as u32);
+                match want {
+                    None => assert_eq!(
+                        value.to_bits(),
+                        TAG_UNDEFINED,
+                        "/{pattern}/ over {subject:?}: group {index} must be unset"
+                    ),
+                    Some(want) => assert_eq!(
+                        bytes(value),
+                        want.as_bytes(),
+                        "/{pattern}/ over {subject:?}: group {index}"
+                    ),
+                }
+            }
+        }
+    }
+}

crates/perry-runtime/src/regex/perex_dispatch.rs

@@ -94,10 +94,17 @@ pub(crate) fn call_one(
     result
 }
 
+#[cfg(test)]
+thread_local! {
+    /// `Get(R, "exec")` lookups `execute` performed on this thread.
+    pub(crate) static EXEC_LOOKUPS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+}
+
 /// RegExpExec with operation-owned limits. Lookup happens on every iteration;
 /// a callback may replace exec or recompile the receiver before the next one.
 /// Only the known builtin may omit materialization for a boolean test.
-/// `reuse` is consulted only on the builtin path, after the observable lookup.
+/// `reuse` is consulted only on the builtin path, after the lookup or after
+/// proving the lookup would reach the builtin without running anything.
 pub(crate) fn execute(
     receiver: &RuntimeHandle<'_>,
     input: &RuntimeHandle<'_>,
@@ -111,37 +118,63 @@ pub(crate) fn execute(
     require_object(receiver.get_nanbox_f64())?;
     input.with_mut_ptr::<StringHeader, _>(|input| crate::string::js_string_addref(input));
     let scope = RuntimeHandleScope::new();
-    let method = scope.root_nanbox_f64(get(receiver, b"exec")?);
+    // A RegExp whose own properties, prototype and `exec` are the untouched
+    // builtins reaches the builtin exec without running any code, so the Get
+    // is unobservable and is skipped. Through the generic property path it was
+    // about half of every `test` call (#10166). Anything else takes the Get.
+    let receiver_ptr =
+        crate::value::js_nanbox_get_pointer(receiver.get_nanbox_f64()) as *mut RegExpHeader;
+    let known_builtin = super::is_valid_regex_ptr(receiver_ptr)
+        && crate::object::regex_proto_thunks::regexp_view_uses_builtin(receiver.get_nanbox_f64());
+    if !known_builtin {
+        #[cfg(test)]
+        EXEC_LOOKUPS.with(|lookups| lookups.set(lookups.get() + 1));
+        let method = scope.root_nanbox_f64(get(receiver, b"exec")?);
+        if let Some(result) = execute_override(&scope, &method, receiver, input)? {
+            return Ok(result);
+        }
+    }
+    let re = crate::value::js_nanbox_get_pointer(receiver.get_nanbox_f64()) as *mut RegExpHeader;
+    if !super::is_valid_regex_ptr(re) {
+        return Err(EngineError::Type(
+            "RegExp builtin exec requires a RegExp receiver",
+        ));
+    }
+    // `execute_with_resources` roots both before it allocates.
+    input
+        .with_const_ptr::<StringHeader, _>(|input| {
+            api::execute_with_resources(re, input, materialize, budget, memory, poll, reuse)
+        })
+        .map(|result| result.map(ExecResult::Builtin))
+}
+
+/// The observable half of RegExpExec: call a looked-up `exec` that is not the
+/// builtin, and check what it returns. `Ok(None)` means the builtin runs.
+fn execute_override(
+    scope: &RuntimeHandleScope,
+    method: &RuntimeHandle<'_>,
+    receiver: &RuntimeHandle<'_>,
+    input: &RuntimeHandle<'_>,
+) -> Result<Option<Option<ExecResult>>, EngineError> {
     let callable = crate::proxy::proxy_wraps_callable(method.get_nanbox_f64());
     let builtin =
         crate::object::regex_proto_thunks::is_builtin_regexp_exec(method.get_nanbox_f64());
     if callable && !builtin {
         let argument = scope.root_nanbox_f64(
             input.with_const_ptr::<StringHeader, _>(|input| js_nanbox_string(input as i64)),
         );
-        let value = call_one(&method, receiver, &argument)?;
+        let value = call_one(method, receiver, &argument)?;
         if value.to_bits() == TAG_NULL {
-            return Ok(None);
+            return Ok(Some(None));
         }
         if !crate::proxy::reflect_value_is_object(value) {
             return Err(EngineError::Type(
                 "RegExp exec method must return an object or null",
             ));
         }
-        return Ok(Some(ExecResult::Override(value)));
-    }
-    let re = crate::value::js_nanbox_get_pointer(receiver.get_nanbox_f64()) as *mut RegExpHeader;
-    if !super::is_valid_regex_ptr(re) {
-        return Err(EngineError::Type(
-            "RegExp builtin exec requires a RegExp receiver",
-        ));
+        return Ok(Some(Some(ExecResult::Override(value))));
     }
-    // `execute_with_resources` roots both before it allocates.
-    input
-        .with_const_ptr::<StringHeader, _>(|input| {
-            api::execute_with_resources(re, input, materialize, budget, memory, poll, reuse)
-        })
-        .map(|result| result.map(ExecResult::Builtin))
+    Ok(None)
 }
 
 pub(crate) fn to_string(value: &RuntimeHandle<'_>) -> Result<*mut StringHeader, EngineError> {

crates/perry-runtime/src/regex/perex_memory.rs

@@ -47,6 +47,28 @@ impl MemoryBudget {
     }
 }
 
+/// Bytes charged to an operation's limit for storage it holds without
+/// allocating, such as match slots kept inline. The limit and peak see them as
+/// they would a buffer's; the collector is not told, since nothing is on its
+/// heap or the native heap.
+pub(crate) struct Charge<'a> {
+    budget: &'a MemoryBudget,
+    bytes: usize,
+}
+impl<'a> Charge<'a> {
+    pub(crate) fn new(budget: &'a MemoryBudget, bytes: usize) -> Result<Self, StorageError> {
+        let live = budget.check(bytes)?;
+        budget.live.set(live);
+        budget.peak.set(budget.peak.get().max(live));
+        Ok(Self { budget, bytes })
+    }
+}
+impl Drop for Charge<'_> {
+    fn drop(&mut self) {
+        self.budget.live.set(self.budget.live.get() - self.bytes);
+    }
+}
+
 /// Account a stable native allocation whose GC-bearing slots are separately
 /// registered with the host's mutable root scanner before this can collect.
 pub(super) struct Reservation<'a> {

crates/perry-runtime/src/regex/perex_runtime.rs

@@ -2,7 +2,7 @@
 //! is a UTF-16 span. Collection and cancellation occur outside resource views.
 
 use super::flags::CanonicalFlags;
-use super::perex_memory::{Buffer, MemoryBudget, StorageError};
+use super::perex_memory::{Buffer, Charge, MemoryBudget, StorageError};
 use super::perex_owner::{BuildError, GcProgram, OwnerError};
 use crate::gc::RuntimeHandleScope;
 use perex::binding::{
@@ -106,18 +106,79 @@ pub(crate) fn compile<'scope, S: ImmutableSubject<Error = OwnerError>>(
     }
 }
 
+/// Match slots a search per call needs, held inline when they fit: such a call
+/// allocates nothing and notes no external bytes. A heap buffer was about a
+/// tenth of every short `test` (#10166). Inline slots are still charged to the
+/// operation's limit, exactly as a buffer of the same count is. Past `N`
+/// slots, and for any growth, they are a heap buffer as before.
+pub(crate) enum Slots<'a, T: Copy + Default, const N: usize> {
+    /// The slots, how many are in use, and their charge to the limit, which
+    /// is released when they are dropped.
+    Inline {
+        slots: [T; N],
+        count: usize,
+        _charge: Charge<'a>,
+    },
+    Heap(Buffer<'a, T>),
+}
+
+impl<'a, T: Copy + Default, const N: usize> Slots<'a, T, N> {
+    fn new(memory: &'a MemoryBudget, count: usize) -> Result<Self, StorageError> {
+        if count <= N {
+            let bytes = count
+                .checked_mul(std::mem::size_of::<T>())
+                .ok_or(StorageError::Limit)?;
+            let charge = Charge::new(memory, bytes)?;
+            Ok(Self::Inline {
+                slots: [T::default(); N],
+                count,
+                _charge: charge,
+            })
+        } else {
+            Buffer::new(memory, count).map(Self::Heap)
+        }
+    }
+}
+
+impl<T: Copy + Default, const N: usize> std::ops::Deref for Slots<'_, T, N> {
+    type Target = [T];
+    fn deref(&self) -> &[T] {
+        match self {
+            Self::Inline { slots, count, .. } => &slots[..*count],
+            Self::Heap(buffer) => buffer,
+        }
+    }
+}
+
+impl<T: Copy + Default, const N: usize> std::ops::DerefMut for Slots<'_, T, N> {
+    fn deref_mut(&mut self) -> &mut [T] {
+        match self {
+            Self::Inline { slots, count, .. } => &mut slots[..*count],
+            Self::Heap(buffer) => buffer,
+        }
+    }
+}
+
+/// Registers a program can have and still search without allocating. Frames
+/// and undo entries start empty and only grow through `rebuffer`, so they are
+/// never inline.
+const INLINE_REGISTERS: usize = 32;
+/// Capture spans an `exec` result can have and still be read without
+/// allocating.
+const INLINE_CAPTURES: usize = 16;
+
 struct MatchBuffers<'a> {
-    registers: Buffer<'a, usize>,
-    frames: Buffer<'a, Frame>,
-    undo: Buffer<'a, Undo>,
+    registers: Slots<'a, usize, INLINE_REGISTERS>,
+    frames: Slots<'a, Frame, 0>,
+    undo: Slots<'a, Undo, 0>,
 }
 
 impl<'a> MatchBuffers<'a> {
     fn new(memory: &'a MemoryBudget, size: ScratchRequirements) -> Result<Self, StorageError> {
         Ok(Self {
-            registers: Buffer::new(memory, size.registers)?,
-            frames: Buffer::new(memory, size.frames)?,
-            undo: Buffer::new(memory, size.undo)?,
+            registers: Slots::new(memory, size.registers)?,
+            frames: Slots::new(memory, size.frames)?,
+            undo: Slots::new(memory, size.undo)?,
         })
     }
 }
@@ -142,7 +203,7 @@ pub(crate) enum CaptureMode {
 pub(crate) struct Match<'a> {
     pub(crate) full: Span,
     /// None under Full. All retains unset groups and includes group zero.
-    pub(crate) captures: Option<Buffer<'a, Option<Span>>>,
+    pub(crate) captures: Option<Slots<'a, Option<Span>, INLINE_CAPTURES>>,
 }
 
 fn search_error(error: SearchError<PairError<OwnerError, OwnerError>>) -> EngineError {
@@ -225,7 +286,7 @@ pub(crate) fn find_near<'mem, S: ImmutableSubject<Error = OwnerError>>(
                     CaptureMode::Full => None,
                     CaptureMode::All => {
                         poll()?;
-                        let mut output = Buffer::new(memory, search.capture_count())?;
+                        let mut output = Slots::new(memory, search.capture_count())?;
                         search
                             .copy_captures(&mut output)
                             .map_err(EngineError::Execution)?;