perf(regex): take perex 0.1.7 and lend the thread's scratch to a search (#10166)

Cargo.toml

@@ -399,7 +399,7 @@ chrono = "0.4"
 regex = "1.12"
 aho-corasick = "1.1"
 # The single regular-expression engine, through crates/perry-perex.
-perex = "0.1.4"
+perex = "0.1.7"
 hex = "0.4"
 tempfile = "3"
 itoa = "1.0"

changelog.d/10372-perex-017-lent-scratch.md

@@ -0,0 +1,7 @@
+### Faster
+
+- `RegExp` calls no longer build their match scratch per call. A `.test()` on a short string costs about a third fewer instructions, and `exec` about a quarter fewer, because the thread lends one scratch cell to each search instead of constructing and moving a fresh one (#10166).
+
+### Changed
+
+- The regex engine moves from perex 0.1.4 to 0.1.7. The `v` flag is complete: string members (`[\q{abc|de}]`), the properties of strings such as `\p{RGI_Emoji}`, and set operators such as `[a--b]` all compile and match. A start-anchored pattern now tries only one start.

crates/perry-runtime/src/gc/tests/runtime_roots/perex_execution.rs

@@ -319,9 +319,32 @@ fn perex_host_failures_release_scratch_and_preserve_consumed_work() {
     }
     assert_eq!(memory.live_bytes(), 0);
     assert_eq!(external_side_live_bytes(), before);
-    // `[a]` under `v` compiles now that the engine implements the union
-    // grammar; its set *operators* are what remain unimplemented.
-    for (pattern, flags, work) in [("(", "", 100_000), ("[a--b]", "v", 100_000), ("a", "", 0)] {
+    // perex 0.1.7 completes the `v` grammar: set operators compile too, so
+    // `[a--b]` is a successful difference rather than an Unsupported witness.
+    {
+        let program = compile(&scope, "[a--b]", "v");
+        let input = subject(&scope, b"ba");
+        let result = host::find(
+            &program,
+            &input,
+            0,
+            CaptureMode::All,
+            &mut Budget::new(100_000),
+            &memory,
+            1,
+            &mut host::poll,
+        )
+        .unwrap()
+        .unwrap();
+        // `a` minus `b`: the `b` at 0 is not a member, the `a` at 1 is.
+        assert_eq!(result.full, Span::new(1, 2).unwrap());
+    }
+    assert_eq!(memory.live_bytes(), 0);
+    assert_eq!(external_side_live_bytes(), before);
+    // Compile failures still release scratch and keep the work they charged.
+    // Nothing in the `v` grammar reports Unsupported any more, so the arms
+    // left are a syntax error and an exhausted work budget.
+    for (pattern, flags, work) in [("(", "", 100_000), ("a", "", 0)] {
         let input = subject(&scope, pattern.as_bytes());
         let mut budget = Budget::new(work);
         let result = host::compile(
@@ -341,14 +364,6 @@ fn perex_host_failures_release_scratch_and_preserve_consumed_work() {
                 result,
                 Err(EngineError::Compile(CompileError::WorkLimit))
             ));
-        } else if flags == "v" {
-            assert!(matches!(
-                result,
-                Err(EngineError::Compile(CompileError::Unsupported {
-                    feature: "Unicode sets",
-                    ..
-                }))
-            ));
         } else {
             assert!(matches!(
                 result,

crates/perry-runtime/src/regex/perex_runtime.rs

@@ -159,10 +159,15 @@ impl<T: Copy + Default, const N: usize> std::ops::DerefMut for Slots<'_, T, N> {
     }
 }
 
-/// Registers a program can have and still search without allocating. Frames
-/// and undo entries start empty and only grow through `rebuffer`, so they are
-/// never inline.
-const INLINE_REGISTERS: usize = 32;
+/// Registers a program can have and still search without allocating on the
+/// owned path, which builds its buffers per call. Most programs need far
+/// fewer: `/^[a-z]+_[0-9]+$/` needs 2. Frames and undo entries start empty and
+/// only grow through `rebuffer`, so they are never inline.
+const INLINE_REGISTERS: usize = 8;
+/// Registers the lent cell holds. This array is allocated once per thread, not
+/// per call, so it is sized for the programs a search may bring rather than
+/// for what is cheap to move.
+const LENT_REGISTERS: usize = 32;
 /// Capture spans an `exec` result can have and still be read without
 /// allocating.
 const INLINE_CAPTURES: usize = 16;
@@ -196,6 +201,55 @@ impl ScratchOwner for MatchBuffers<'_> {
     }
 }
 
+/// Scratch a thread lends to one search at a time, instead of building an
+/// owner per call (#10166).
+///
+/// `find_near` built a `MatchBuffers` for every call: a 32-register inline
+/// array zeroed and then moved by value into `Search`, which disassembled to a
+/// 336-byte `memcpy` at every call and measured as a third of a short
+/// `.test()`. Nothing in that scratch depends on the subject, and a search
+/// initializes its own live state, so one cell serves every call on the
+/// thread. The arrays keep whatever size an earlier call needed, so a loop
+/// reaches a steady state that grows nothing and constructs nothing.
+///
+/// No GC pointer is ever stored here: registers are subject offsets, and
+/// frames and undo entries are the engine's own opaque scratch, exactly as in
+/// the owned buffers this replaces (see this module's header).
+struct ScratchCell {
+    registers: [usize; LENT_REGISTERS],
+    frames: Vec<Frame>,
+    undo: Vec<Undo>,
+}
+
+crate::perry_thread_local! {
+    /// Borrowed for one search. A nested regex — a replacer callback that runs
+    /// its own match, or a poll that re-enters — finds the cell borrowed and
+    /// takes the owned path, so two searches never share slots. This is the
+    /// runtime half of the guarantee perex's `ScratchOwner for &mut O` makes at
+    /// compile time within a single frame.
+    ///
+    /// `perry_thread_local!` rather than the raw macro (#7469): every search on
+    /// this thread reads it, so the address belongs in the hot cache instead of
+    /// costing a `_tlv_get_addr` call — the opposite of what this change is for.
+    static LENT_SCRATCH: std::cell::RefCell<ScratchCell> = const {
+        std::cell::RefCell::new(ScratchCell {
+            registers: [0; LENT_REGISTERS],
+            frames: Vec::new(),
+            undo: Vec::new(),
+        })
+    };
+}
+
+/// What a lent attempt produced: an answer, or a reason to run the owned path.
+enum Lent<'mem> {
+    Done(Option<Match<'mem>>, Position),
+    /// The cell was already borrowed, or the search asked for more frames or
+    /// undo entries than it holds. The cell has been grown to the requested
+    /// size, so the next call starts big enough; this call runs the owned path
+    /// from its entry budget, exactly as it would have without lending.
+    Fallback,
+}
+
 #[derive(Clone, Copy)]
 pub(crate) enum CaptureMode {
     /// Test/search need only the full match, without allocating output slots.
@@ -243,6 +297,105 @@ pub(crate) fn find<'mem, S: ImmutableSubject<Error = OwnerError>>(
 /// `near` must come from a search or reader over this same binding. Another
 /// string with an identical layout cannot be detected and would give wrong
 /// answers, so callers keep a position only as long as the binding it came from.
+/// One search over lent scratch. Returns `Fallback` without an answer when the
+/// scratch cannot serve this search; the caller then runs the owned path.
+#[allow(clippy::too_many_arguments)]
+fn find_near_lent<'mem, S: ImmutableSubject<Error = OwnerError>>(
+    resources: &BoundResources<'_, GcProgram<'_>, S>,
+    registers: usize,
+    start: usize,
+    near: Option<Position>,
+    mode: CaptureMode,
+    budget: &mut Budget,
+    memory: &'mem MemoryBudget,
+    quantum: usize,
+    poll: &mut impl FnMut() -> Result<(), EngineError>,
+) -> Result<Lent<'mem>, EngineError> {
+    LENT_SCRATCH.with(|cell| {
+        let Ok(mut cell) = cell.try_borrow_mut() else {
+            return Ok(Lent::Fallback);
+        };
+        let cell = &mut *cell;
+        // Charged exactly like the owner it replaces: the operation's limit
+        // sees the slots a search may use, whether or not they were allocated
+        // for it. The thread keeps the memory; the operation only borrows it.
+        let bytes = registers
+            .checked_mul(std::mem::size_of::<usize>())
+            .and_then(|n| {
+                n.checked_add(
+                    cell.frames
+                        .len()
+                        .checked_mul(std::mem::size_of::<Frame>())?,
+                )
+            })
+            .and_then(|n| n.checked_add(cell.undo.len().checked_mul(std::mem::size_of::<Undo>())?))
+            .ok_or(StorageError::Limit)?;
+        let _charge = Charge::new(memory, bytes)?;
+        let scratch = Scratch {
+            registers: &mut cell.registers[..registers],
+            frames: &mut cell.frames[..],
+            undo: &mut cell.undo[..],
+        };
+        poll()?;
+        let mut search = match near {
+            Some(near) => Search::new_near(resources, start, near, scratch, *budget),
+            None => Search::new(resources, start, scratch, *budget),
+        }
+        .map_err(search_error)?;
+        loop {
+            let result = search.advance(quantum);
+            *budget = Budget::new(search.remaining_work());
+            match result {
+                Ok(Progress::NoMatch) => return Ok(Lent::Done(None, search.position())),
+                Ok(Progress::Matched) => {
+                    let full = search
+                        .capture(0)
+                        .map_err(EngineError::Execution)?
+                        .ok_or(EngineError::Execution(ExecError::InvalidProgram))?;
+                    let captures = match mode {
+                        CaptureMode::Full => None,
+                        CaptureMode::All => {
+                            poll()?;
+                            let mut output = Slots::new(memory, search.capture_count())?;
+                            search
+                                .copy_captures(&mut output)
+                                .map_err(EngineError::Execution)?;
+                            Some(output)
+                        }
+                    };
+                    let position = search.position();
+                    return Ok(Lent::Done(Some(Match { full, captures }), position));
+                }
+                Ok(Progress::Pending) => poll()?,
+                Err(SearchError::Execution(ExecError::Frames | ExecError::Undo)) => {
+                    // Grow the cell for the next call and let this one run the
+                    // owned path, which charges the whole search once from the
+                    // caller's entry budget. Rebuffering in place would need a
+                    // second borrow of the cell the search already holds.
+                    let required = search.required_scratch();
+                    drop(search);
+                    if crate::hot_diag::regex_on() {
+                        crate::hot_diag::regex_with(|d| d.perex_scratch_grows += 1);
+                    }
+                    let frames = required
+                        .frames
+                        .max(cell.frames.len().saturating_mul(2))
+                        .max(8);
+                    let undo = required.undo.max(cell.undo.len().saturating_mul(2)).max(16);
+                    if frames > cell.frames.len() {
+                        cell.frames.resize(frames, Frame::default());
+                    }
+                    if undo > cell.undo.len() {
+                        cell.undo.resize(undo, Undo::default());
+                    }
+                    return Ok(Lent::Fallback);
+                }
+                Err(error) => return Err(search_error(error)),
+            }
+        }
+    })
+}
+
 pub(crate) fn find_near<'mem, S: ImmutableSubject<Error = OwnerError>>(
     program: &BoundProgram<GcProgram<'_>>,
     subject: &BoundSubject<S>,
@@ -269,6 +422,19 @@ pub(crate) fn find_near<'mem, S: ImmutableSubject<Error = OwnerError>>(
         frames: 0,
         undo: 0,
     };
+    // Lend the thread's scratch first: a search that fits it constructs and
+    // moves nothing (#10166). Anything the cell cannot serve falls through to
+    // the owned buffers below with the budget it entered on.
+    if registers <= LENT_REGISTERS {
+        let entry = *budget;
+        match find_near_lent(
+            &resources, registers, start, near, mode, budget, memory, quantum, poll,
+        )? {
+            Lent::Done(found, position) => return Ok((found, position)),
+            Lent::Fallback => *budget = entry,
+        }
+    }
+
     poll()?;
     let buffers = MatchBuffers::new(memory, size)?;
     let mut search = match near {

scripts/gc_runtime_root_holders.json

@@ -955,6 +955,12 @@
       "verdict": "test_only",
       "why": "#10164: #[cfg(test)] Cell<u64> counting how many searches resumed from a recorded cross-call position, so tests can tell the position was used. A count, never an address, and absent from shipped binaries."
     },
+    {
+      "file": "crates/perry-runtime/src/regex/perex_runtime.rs",
+      "name": "LENT_SCRATCH",
+      "verdict": "not_a_gc_pointer",
+      "why": "Per-thread match scratch lent to one search at a time (#10166): registers are subject offsets in UTF-16 units, frames and undo entries are perex's own opaque evaluator scratch. No GC pointer is ever stored, exactly as for the per-call MatchBuffers it replaces (this module's header and perex_memory.rs). The cell is borrowed for one search and holds no program, subject or result: those stay in GcProgram/HeapSubject roots and in the operation's MemoryBudget-charged slots."
+    },
     {
       "file": "crates/perry-runtime/src/regex/perex_split.rs",
       "name": "FORWARD_SPLITS",