fix(string): apply context-dependent final sigma lowercasing

changelog.d/10226-final-sigma.md

@@ -0,0 +1,6 @@
+Fix Greek final-sigma casing in `String.prototype.toLowerCase()`: a capital
+sigma preceded by a cased letter and not followed by one now becomes `ς`,
+skipping Unicode case-ignorable characters on either side. Whole-string
+lowercasing preserves this context while bounded decoding retains lone
+surrogates and malformed payload boundaries. The ASCII fast path remains
+unchanged.

crates/perry-runtime/src/string/case_tests.rs

@@ -0,0 +1,77 @@
+//! Context-dependent default casing (#10116).
+
+use super::*;
+
+#[test]
+fn lowercase_final_sigma_uses_cased_and_case_ignorable_context() {
+    // Expected strings were checked against the pinned Node oracle. In
+    // particular, Case_Ignorable characters must be skipped on BOTH sides;
+    // U+0345 is also Cased, so testing Cased first would be incorrect.
+    let cases = [
+        ("ΑΣ", "ας"),
+        ("ΟΔΥΣΣΕΥΣ", "οδυσσευς"),
+        ("Σ", "σ"),
+        ("ΣΣ", "σς"),
+        ("ΑΣΑ", "ασα"),
+        ("AΣ! AΣB", "aς! aσb"),
+        ("1Σ", "1σ"),
+        ("中Σ", "中σ"),
+        ("AΣ1", "aς1"),
+        ("AΣ中", "aς中"),
+        ("A\u{301}Σ", "a\u{301}ς"),
+        ("AΣ\u{301}", "aς\u{301}"),
+        ("AΣ\u{301}B", "aσ\u{301}b"),
+        ("\u{301}Σ", "\u{301}σ"),
+        ("A'Σ", "a'ς"),
+        ("AΣ'B", "aσ'b"),
+        ("AΣ\u{200d}", "aς\u{200d}"),
+        ("AΣ\u{200d}B", "aσ\u{200d}b"),
+        ("\u{345}Σ", "\u{345}σ"),
+        ("A\u{345}Σ", "a\u{345}ς"),
+        ("AΣ\u{345}", "aς\u{345}"),
+        ("AΣ\u{345}B", "aσ\u{345}b"),
+        ("𐐀Σ", "𐐨ς"),
+        ("AΣ𐐀", "aσ𐐨"),
+        ("😀Σ", "😀σ"),
+        ("AΣ😀", "aς😀"),
+        ("İΣ", "i\u{307}ς"),
+        ("AΣİ", "aσi\u{307}"),
+    ];
+    for (input, expected) in cases {
+        let source = js_string_from_bytes(input.as_ptr(), input.len() as u32);
+        let result = js_string_to_lower_case(source);
+        assert_eq!(string_as_str(result), expected, "input: {input:?}");
+        assert_eq!(
+            unsafe { (*result).utf16_len },
+            expected.encode_utf16().count() as u32
+        );
+    }
+}
+
+#[test]
+fn lowercase_final_sigma_preserves_lone_surrogate_boundaries() {
+    for surrogate in [[0xED, 0xA0, 0x80], [0xED, 0xBF, 0xBF]] {
+        for (before, after, lower_before, lower_after) in [
+            ("AΣ", "ΣA", "aς", "σa"),
+            ("A", "Σ", "a", "σ"),
+            ("", "AΣ", "", "aς"),
+            ("AΣ\u{301}", "B", "aς\u{301}", "b"),
+        ] {
+            let input = [before.as_bytes(), &surrogate, after.as_bytes()].concat();
+            let expected = [lower_before.as_bytes(), &surrogate, lower_after.as_bytes()].concat();
+            let source = js_string_from_wtf8_bytes(input.as_ptr(), input.len() as u32);
+            let result = js_string_to_lower_case(source);
+            let bytes =
+                unsafe { slice::from_raw_parts(string_data(result), (*result).byte_len as usize) };
+            assert_eq!(bytes, expected);
+            assert_ne!(
+                unsafe { (*result).flags } & STRING_FLAG_HAS_LONE_SURROGATES,
+                0
+            );
+            assert_eq!(
+                unsafe { (*result).utf16_len },
+                compute_utf16_len_wtf8(&expected)
+            );
+        }
+    }
+}

crates/perry-runtime/src/string/mod.rs

@@ -124,6 +124,8 @@ pub(crate) mod suffix_cursor;
 pub(crate) mod trim_cache;
 mod utf16_count;
 
+#[cfg(test)]
+mod case_tests;
 #[cfg(test)]
 mod slice_tests;
 #[cfg(test)]

crates/perry-runtime/src/string/slice_ops.rs

@@ -327,14 +327,12 @@ pub extern "C" fn js_string_trim_end(s: *const StringHeader) -> *mut StringHeade
 
 /// Unicode case conversion over the raw payload (#6085).
 ///
-/// `str::to_lowercase`/`to_uppercase` iterate `chars()`, which reads
-/// continuation bytes past an exact-sized payload ending in a truncated
-/// multi-byte lead. Decode with the bounded `wtf8_step` instead: sequences that
-/// form a real Unicode scalar get the full `char` case mapping (identical
-/// output for well-formed input, including multi-char expansions like `ß`→`SS`),
-/// while a lone surrogate or a truncated/invalid sequence is copied through
-/// VERBATIM — which also preserves the WTF-8 round-trip (#4793) that the old
-/// `from_utf8_unchecked` path only got by accident.
+/// Whole-string lowercasing handles context-dependent Greek sigma (#10116).
+/// Only validated UTF-8 may enter Rust's string iterators: unchecked views
+/// could read past an exact-sized payload ending in a truncated lead (#6085).
+/// For WTF-8, the bounded decoder builds scalar runs for lowercasing; lone
+/// surrogates and undecodable bytes are copied verbatim and end the context.
+/// Uppercasing remains context-free, including expansions such as `ß`→`SS`.
 fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader {
     if !is_valid_string_ptr(s) {
         return js_string_from_bytes(ptr::null(), 0);
@@ -365,7 +363,15 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader {
         return js_string_from_bytes_known_utf16(out.as_ptr(), len, len, 0);
     }
 
+    if !upper {
+        if let Ok(text) = std::str::from_utf8(bytes) {
+            let out = text.to_lowercase();
+            return js_string_from_bytes(out.as_ptr(), out.len() as u32);
+        }
+    }
+
     let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
+    let mut lowercase_run = String::new();
     let mut has_lone_surrogate = false;
     let mut buf = [0u8; 4];
     let mut i = 0usize;
@@ -387,12 +393,16 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader {
                         out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
                     }
                 } else {
-                    for c in ch.to_lowercase() {
-                        out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes());
-                    }
+                    // Keep the original scalars together so Unicode's Cased
+                    // and Case_Ignorable context is available on both sides.
+                    lowercase_run.push(ch);
                 }
             }
             None => {
+                if !lowercase_run.is_empty() {
+                    out.extend_from_slice(lowercase_run.to_lowercase().as_bytes());
+                    lowercase_run.clear();
+                }
                 // Lone surrogate / truncated / stray continuation byte: copy the
                 // raw bytes so the payload round-trips unchanged.
                 if (0xD800..=0xDFFF).contains(&cp) {
@@ -403,6 +413,9 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader {
         }
         i = end;
     }
+    if !lowercase_run.is_empty() {
+        out.extend_from_slice(lowercase_run.to_lowercase().as_bytes());
+    }
     if has_lone_surrogate || unsafe { (*s).flags } & STRING_FLAG_HAS_LONE_SURROGATES != 0 {
         return js_string_from_wtf8_bytes(out.as_ptr(), out.len() as u32);
     }

crates/perry-runtime/src/string/tests_guard_page.rs

@@ -453,3 +453,29 @@ fn case_convert_rejects_the_aggregate_ascii_lie() {
         "É (C3 89) + A + the raw truncated lead byte — NOT the input echoed back unmapped"
     );
 }
+
+#[test]
+fn lowercase_final_sigma_keeps_truncated_tail_bounded() {
+    // Every payload is twelve bytes and ends exactly at the guard page. The
+    // scalar run before an incomplete lead still needs final-sigma context.
+    for (input, expected) in [
+        (
+            b"    ABCDE\xCE\xA3\xF0".as_slice(),
+            b"    abcde\xCF\x82\xF0".as_slice(),
+        ),
+        (
+            b"    ABCD\xCE\xA3\xE2\x82".as_slice(),
+            b"    abcd\xCF\x82\xE2\x82".as_slice(),
+        ),
+        (
+            b"    ABC\xCE\xA3\xF0\x9F\x98".as_slice(),
+            b"    abc\xCF\x82\xF0\x9F\x98".as_slice(),
+        ),
+    ] {
+        let guarded = GuardedString::new(input);
+        let result = js_string_to_lower_case(guarded.ptr());
+        let bytes =
+            unsafe { slice::from_raw_parts(string_data(result), (*result).byte_len as usize) };
+        assert_eq!(bytes, expected);
+    }
+}

test-files/test_gap_10090_string_case_ascii_fastpath.ts

@@ -58,12 +58,8 @@ console.log("default-I-lower:" + "I".toLowerCase()); // "i"
 console.log("i-with-dot-lower:" + JSON.stringify("İ".toLowerCase()));
 console.log("i-with-dot-lower-units:" + JSON.stringify(units("İ".toLowerCase())));
 
-// NOTE: Greek final sigma (context-dependent Σ -> ς vs σ) is intentionally
-// NOT covered here. It is a pre-existing gap in the untouched scalar path
-// (Rust's char::to_lowercase() has no notion of the conditional Final_Sigma
-// rule) unrelated to the ASCII fast path added by this file's issue, and
-// asserting Node's correct output here would fail on main regardless of this
-// fix. Tracked separately as #10116.
+// Greek final sigma (context-dependent Σ -> ς vs σ) is covered separately
+// in test_gap_10116_final_sigma.ts.
 
 // ---- Cherokee (Unicode 8.0 added case pairs) ----
 console.log("cherokee-lower:" + "Ꭰ".toLowerCase()); // U+AB70

test-files/test_gap_10116_final_sigma.ts

@@ -0,0 +1,40 @@
+// #10116: context-sensitive default lowercasing, including WTF-8 boundaries.
+function lower(input: string): string { return input.toLowerCase(); }
+const cases = [
+  ["ΑΣ", "ας"],
+  ["ΟΔΥΣΣΕΥΣ", "οδυσσευς"],
+  ["Σ", "σ"],
+  ["ΣΣ", "σς"],
+  ["ΑΣΑ", "ασα"],
+  ["AΣ! AΣB", "aς! aσb"],
+  ["1Σ", "1σ"],
+  ["中Σ", "中σ"],
+  ["AΣ1", "aς1"],
+  ["AΣ中", "aς中"],
+  ["A\u{301}Σ", "a\u{301}ς"],
+  ["AΣ\u{301}", "aς\u{301}"],
+  ["AΣ\u{301}B", "aσ\u{301}b"],
+  ["\u{301}Σ", "\u{301}σ"],
+  ["A'Σ", "a'ς"],
+  ["AΣ'B", "aσ'b"],
+  ["AΣ\u{200d}", "aς\u{200d}"],
+  ["AΣ\u{200d}B", "aσ\u{200d}b"],
+  ["\u{345}Σ", "\u{345}σ"],
+  ["A\u{345}Σ", "a\u{345}ς"],
+  ["AΣ\u{345}", "aς\u{345}"],
+  ["AΣ\u{345}B", "aσ\u{345}b"],
+  ["𐐀Σ", "𐐨ς"],
+  ["AΣ𐐀", "aσ𐐨"],
+  ["😀Σ", "😀σ"],
+  ["AΣ😀", "aς😀"],
+  ["İΣ", "i\u{307}ς"],
+  ["AΣİ", "aσi\u{307}"],
+  ["AΣ\ud800ΣA", "aς\ud800σa"],
+  ["A\ud800Σ", "a\ud800σ"],
+  ["\udfffAΣ", "\udfffaς"],
+  ["AΣ\u0301\udfffB", "aς\u0301\udfffb"],
+];
+for (let i = 0; i < cases.length; i++) {
+  const actual = lower(cases[i][0]);
+  console.log(i, actual === cases[i][1]);
+}