perf(runtime): add ASCII fast path to toLowerCase/toUpperCase

changelog.d/10117-string-case-ascii-fastpath.md

@@ -0,0 +1,22 @@
+### `toLowerCase`/`toUpperCase` gained an ASCII fast path (#10090)
+
+`case_convert` (the shared implementation behind `String.prototype.toLowerCase`
+and `toUpperCase`) previously ran every input — including pure ASCII — through
+a scalar `wtf8_step` decode, a per-character `char::to_lowercase()` /
+`to_uppercase()` iterator, and a re-encode loop. On a 1M-character all-ASCII
+string this cost 30-33x what Node takes for the same input.
+
+`case_convert` now checks whether the input is pure ASCII with a real
+per-byte scan (`bytes.is_ascii()`) and, if so, produces the result with a
+single vectorizable `to_ascii_lowercase()`/`to_ascii_uppercase()` byte-table
+transform instead. Everything else — Unicode special casing (`ß`→`SS`,
+Cherokee, Deseret, the default-locale `İ`→`i` + combining dot, Greek final
+sigma), WTF-8 lone-surrogate round-tripping, and locale-aware casing in
+`locale.rs` — is unchanged and continues through the original scalar loop.
+
+The ASCII check deliberately does **not** reuse the existing `is_ascii_string`
+helper (an O(1) `byte_len == utf16_len` proxy used elsewhere in this file):
+that aggregate can be true for malformed WTF-8 where a stray continuation
+byte and a truncated multi-byte lead cancel out in the unit count, even
+though the bytes are not ASCII. A regression test
+(`case_convert_rejects_the_aggregate_ascii_lie`) locks this in.

crates/perry-runtime/src/string/slice_ops.rs

@@ -389,6 +389,31 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader {
         return js_string_from_bytes(ptr::null(), 0);
     }
     let bytes = unsafe { slice::from_raw_parts(string_data(s), (*s).byte_len as usize) };
+
+    // ASCII fast path (#10090): default-locale ASCII case mapping is
+    // context-free and strictly 1-byte-in/1-byte-out, so skip the scalar
+    // wtf8_step decode / per-char to_lowercase()/to_uppercase() iterator
+    // construction / re-encode loop entirely and let `to_ascii_lowercase`/
+    // `to_ascii_uppercase` do a vectorizable byte-table transform instead.
+    //
+    // Must gate on `bytes.is_ascii()` (a real per-byte scan), NOT the
+    // `is_ascii_string(s)` `byte_len == utf16_len` AGGREGATE proxy used
+    // elsewhere for O(1) checks: that aggregate can lie for malformed WTF-8,
+    // where a stray continuation byte (0 UTF-16 units) and a truncated
+    // multi-byte lead (2 units) cancel out to look ASCII while containing
+    // non-ASCII bytes (see `split_parts_get_metadata_from_their_own_bytes`).
+    // A genuinely all-ASCII input can never carry a lone surrogate, so the
+    // result's flags are trivially 0 and its utf16_len == its byte_len.
+    if bytes.is_ascii() {
+        let out = if upper {
+            bytes.to_ascii_uppercase()
+        } else {
+            bytes.to_ascii_lowercase()
+        };
+        let len = out.len() as u32;
+        return js_string_from_bytes_known_utf16(out.as_ptr(), len, len, 0);
+    }
+
     let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
     let mut has_lone_surrogate = false;
     let mut buf = [0u8; 4];

crates/perry-runtime/src/string/tests.rs

@@ -1393,3 +1393,59 @@ fn header_str_checked_matches_from_utf8_on_every_payload_class() {
 fn string_as_bytes_for_test<'a>(s: *const StringHeader) -> &'a [u8] {
     unsafe { slice::from_raw_parts(string_data(s), (*s).byte_len as usize) }
 }
+
+/// ASCII fast path (#10090): `case_convert` must produce byte-identical
+/// results to the pre-fast-path scalar decode loop for pure-ASCII input,
+/// including empty strings, single characters, and input already in the
+/// target case. `utf16_len` must equal `byte_len` and `flags` must be 0 —
+/// an all-ASCII payload can never carry `STRING_FLAG_HAS_LONE_SURROGATES`.
+#[test]
+fn ascii_fast_path_basic_case_conversion() {
+    for input in [
+        "",
+        "a",
+        "A",
+        "aBcD1234EfGh",
+        "already lower",
+        "ALREADY UPPER",
+    ] {
+        let s = js_string_from_bytes(input.as_ptr(), input.len() as u32);
+        let lower = js_string_to_lower_case(s);
+        let upper = js_string_to_upper_case(s);
+        assert_eq!(string_as_str(lower), input.to_ascii_lowercase());
+        assert_eq!(string_as_str(upper), input.to_ascii_uppercase());
+        for out in [lower, upper] {
+            unsafe {
+                assert_eq!((*out).utf16_len, (*out).byte_len);
+                assert_eq!((*out).flags, 0);
+            }
+        }
+    }
+}
+
+/// A lone surrogate after an ASCII prefix must still take the scalar path:
+/// `bytes.is_ascii()` is false (the WTF-8 encoding of a lone surrogate uses
+/// bytes >= 0x80), so the ASCII fast path is not taken, the surrogate bytes
+/// round-trip verbatim, and `STRING_FLAG_HAS_LONE_SURROGATES` survives.
+#[test]
+fn ascii_prefix_with_trailing_lone_surrogate_preserves_flag_and_bytes() {
+    // "ABC" + WTF-8 lone high surrogate U+D800 (ED A0 80).
+    let bytes = [b'A', b'B', b'C', 0xED, 0xA0, 0x80];
+    let s = js_string_from_wtf8_bytes(bytes.as_ptr(), bytes.len() as u32);
+    assert_ne!(unsafe { (*s).flags } & STRING_FLAG_HAS_LONE_SURROGATES, 0);
+    assert!(!string_as_bytes_for_test(s).is_ascii());
+
+    for (out, expect_ascii) in [
+        (js_string_to_lower_case(s), [b'a', b'b', b'c']),
+        (js_string_to_upper_case(s), [b'A', b'B', b'C']),
+    ] {
+        let out_bytes = string_as_bytes_for_test(out);
+        assert_eq!(&out_bytes[..3], &expect_ascii);
+        assert_eq!(&out_bytes[3..], &[0xED, 0xA0, 0x80]);
+        assert_eq!(
+            unsafe { (*out).flags } & STRING_FLAG_HAS_LONE_SURROGATES,
+            STRING_FLAG_HAS_LONE_SURROGATES,
+            "the lone surrogate must keep the result flagged"
+        );
+    }
+}

crates/perry-runtime/src/string/tests_guard_page.rs

@@ -414,3 +414,42 @@ fn astral_and_truncated_astral_lead() {
     let _ = js_string_char_code_at(ts, 2);
     let _ = js_string_to_char_array(crate::value::js_nanbox_string(ts as i64).to_bits() as i64);
 }
+
+/// The ASCII fast path added for #10090 MUST gate on `bytes.is_ascii()` (a
+/// real per-byte scan), not on `is_ascii_string(s)` (the `byte_len ==
+/// utf16_len` AGGREGATE proxy already known to lie — see
+/// `split_parts_get_metadata_from_their_own_bytes`).
+///
+/// `[0xC3, 0xA9, b'a', 0xF0]` is "é" + "a" + a truncated 4-byte lead. Its
+/// UTF-16 unit total happens to equal its byte length (1 + 1 + 2 == 4), so
+/// `is_ascii_string` wrongly reports true, even though the payload is not
+/// ASCII. A gate on the aggregate would route this through
+/// `to_ascii_uppercase()`, which touches only `a`-`z`/`A`-`Z` bytes and
+/// leaves 0xC3/0xA9 untouched — silently skipping the real Unicode mapping
+/// `é` → `É` (`C3 A9` → `C3 89`). The correct byte-level gate falls back to
+/// the scalar `wtf8_step` loop, which maps `é`/`a` and copies the truncated
+/// lead byte through verbatim, flush against the guard page so any
+/// out-of-bounds read on that fallback also faults.
+#[test]
+fn case_convert_rejects_the_aggregate_ascii_lie() {
+    let g = GuardedString::new(&[0xC3, 0xA9, b'a', 0xF0]);
+    let s = g.ptr();
+    assert!(
+        is_ascii_string(s),
+        "precondition: the aggregate byte_len == utf16_len check misfires"
+    );
+    let bytes = unsafe { slice::from_raw_parts(string_data(s), (*s).byte_len as usize) };
+    assert!(
+        !bytes.is_ascii(),
+        "precondition: the payload is NOT actually pure ASCII"
+    );
+
+    let upper = js_string_to_upper_case(s);
+    let upper_bytes =
+        unsafe { slice::from_raw_parts(string_data(upper), (*upper).byte_len as usize) };
+    assert_eq!(
+        upper_bytes,
+        &[0xC3, 0x89, b'A', 0xF0],
+        "É (C3 89) + A + the raw truncated lead byte — NOT the input echoed back unmapped"
+    );
+}

test-files/test_gap_10090_string_case_ascii_fastpath.ts

@@ -0,0 +1,90 @@
+// Gap test for #10090: toLowerCase/toUpperCase gained an ASCII fast path in
+// `case_convert` (crates/perry-runtime/src/string/slice_ops.rs) that skips
+// the scalar wtf8_step decode / char::to_lowercase()-or-to_uppercase()
+// iterator / re-encode loop when every byte of the input is ASCII, doing a
+// plain byte-table transform instead.
+//
+// The fast path must not change behavior for anything it does not apply to.
+// This file exercises exactly the boundary cases called out in the issue:
+// multi-char Unicode special casing (ß, Cherokee, Deseret, final sigma, the
+// default-locale İ special case), a string that is ASCII except for one
+// trailing multi-byte character, and an ASCII prefix followed by a lone
+// surrogate. This file is byte-compared with `node --experimental-strip-types`
+// by the gap suite.
+
+function show(label: string, value: unknown): void {
+  console.log(label + ":" + JSON.stringify(value));
+}
+
+// Render a string as its UTF-16 code-unit sequence so a lone surrogate
+// survives JSON.stringify unambiguously (matches test_gap_9409's `units`).
+function units(s: string): number[] {
+  return Array.from({ length: s.length }, (_, i) => s.charCodeAt(i));
+}
+
+// ---- Pure ASCII: fast-path territory ----
+const asciiSamples = [
+  "",
+  "a",
+  "A",
+  "aBcD1234EfGh",
+  "already lower",
+  "ALREADY UPPER",
+  "The Quick Brown Fox Jumps Over The Lazy Dog 0123456789 !@#$%^&*()",
+  "aBcD".repeat(50),
+];
+for (const s of asciiSamples) {
+  show("ascii-lower:" + JSON.stringify(s), s.toLowerCase());
+  show("ascii-upper:" + JSON.stringify(s), s.toUpperCase());
+  show("ascii-lower-len:" + JSON.stringify(s), s.toLowerCase().length);
+  show("ascii-upper-len:" + JSON.stringify(s), s.toUpperCase().length);
+}
+
+// ---- German sharp s: one-to-many default casing, changes .length ----
+console.log("sharp-s-upper:" + "straße".toUpperCase()); // "STRASSE"
+console.log("sharp-s-upper-len:" + "straße".toUpperCase().length); // 7 (was 6)
+console.log("capital-sharp-s-lower:" + "ẞ".toLowerCase()); // "ß"
+
+// ---- Turkic dotted/dotless I must NOT apply under the DEFAULT (non-locale)
+// toLowerCase/toUpperCase - that special casing only applies to
+// toLocaleLowerCase/toLocaleUpperCase("tr"/"az"), which live in locale.rs and
+// are untouched by this fix. ----
+console.log("default-i-upper:" + "i".toUpperCase()); // "I"
+console.log("default-I-lower:" + "I".toLowerCase()); // "i"
+
+// ---- Default-locale one-to-many special casing (SpecialCasing.txt, locale
+// independent): CAPITAL I WITH DOT ABOVE lowercases to "i" + COMBINING DOT
+// ABOVE - two code units, distinct from the Turkish-locale mapping. ----
+console.log("i-with-dot-lower:" + JSON.stringify("İ".toLowerCase()));
+console.log("i-with-dot-lower-units:" + JSON.stringify(units("İ".toLowerCase())));
+
+// NOTE: Greek final sigma (context-dependent Σ -> ς vs σ) is intentionally
+// NOT covered here. It is a pre-existing gap in the untouched scalar path
+// (Rust's char::to_lowercase() has no notion of the conditional Final_Sigma
+// rule) unrelated to the ASCII fast path added by this file's issue, and
+// asserting Node's correct output here would fail on main regardless of this
+// fix. Tracked separately as #10116.
+
+// ---- Cherokee (Unicode 8.0 added case pairs) ----
+console.log("cherokee-lower:" + "Ꭰ".toLowerCase()); // U+AB70
+console.log("cherokee-upper:" + "ꭰ".toUpperCase()); // U+13A0
+
+// ---- Deseret (astral, surrogate pair) ----
+console.log("deseret-lower:" + "𐐀".toLowerCase()); // U+10428 -> 𐐨
+console.log("deseret-lower-units:" + JSON.stringify(units("𐐀".toLowerCase())));
+console.log("deseret-upper:" + "𐐨".toUpperCase()); // U+10400 -> 𐐀
+
+// ---- ASCII except for one trailing multi-byte character ----
+const asciiPlusOne = "hello" + "é"; // "helloé"
+console.log("ascii-plus-one-upper:" + asciiPlusOne.toUpperCase()); // "HELLOÉ"
+console.log("ascii-plus-one-lower:" + asciiPlusOne.toLowerCase()); // "helloé"
+
+// ---- ASCII prefix followed by a lone surrogate: must NOT take the ASCII
+// fast path (bytes.is_ascii() is false), and the lone surrogate must survive
+// verbatim through both directions. ----
+const asciiPlusLone = "ABC\ud800";
+show("ascii-plus-lone-src-units", units(asciiPlusLone));
+show("ascii-plus-lone-lower-units", units(asciiPlusLone.toLowerCase()));
+show("ascii-plus-lone-upper-units", units(asciiPlusLone.toUpperCase()));
+console.log("ascii-plus-lone-lower-len:" + asciiPlusLone.toLowerCase().length);
+console.log("ascii-plus-lone-upper-len:" + asciiPlusLone.toUpperCase().length);