From fdbfa695c7d36be49c41aca6e866ad7eb0a3ad8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 22:56:08 +0200 Subject: [PATCH 1/2] fix(string): apply context-dependent final sigma lowercasing --- changelog.d/10116-final-sigma.md | 6 ++ crates/perry-runtime/src/string/case_tests.rs | 77 +++++++++++++++++++ crates/perry-runtime/src/string/mod.rs | 2 + crates/perry-runtime/src/string/slice_ops.rs | 35 ++++++--- .../src/string/tests_guard_page.rs | 26 +++++++ ...st_gap_10090_string_case_ascii_fastpath.ts | 8 +- test-files/test_gap_10116_final_sigma.ts | 40 ++++++++++ 7 files changed, 177 insertions(+), 17 deletions(-) create mode 100644 changelog.d/10116-final-sigma.md create mode 100644 crates/perry-runtime/src/string/case_tests.rs create mode 100644 test-files/test_gap_10116_final_sigma.ts diff --git a/changelog.d/10116-final-sigma.md b/changelog.d/10116-final-sigma.md new file mode 100644 index 0000000000..099a990a24 --- /dev/null +++ b/changelog.d/10116-final-sigma.md @@ -0,0 +1,6 @@ +Fix Greek final-sigma casing in `String.prototype.toLowerCase()`: a capital +sigma preceded by a cased letter and not followed by one now becomes `ς`, +skipping Unicode case-ignorable characters on either side. Whole-string +lowercasing preserves this context while bounded decoding retains lone +surrogates and malformed payload boundaries. The ASCII fast path remains +unchanged. diff --git a/crates/perry-runtime/src/string/case_tests.rs b/crates/perry-runtime/src/string/case_tests.rs new file mode 100644 index 0000000000..076e29dfb0 --- /dev/null +++ b/crates/perry-runtime/src/string/case_tests.rs @@ -0,0 +1,77 @@ +//! Context-dependent default casing (#10116). + +use super::*; + +#[test] +fn lowercase_final_sigma_uses_cased_and_case_ignorable_context() { + // Expected strings were checked against the pinned Node oracle. In + // particular, Case_Ignorable characters must be skipped on BOTH sides; + // U+0345 is also Cased, so testing Cased first would be incorrect. + let cases = [ + ("ΑΣ", "ας"), + ("ΟΔΥΣΣΕΥΣ", "οδυσσευς"), + ("Σ", "σ"), + ("ΣΣ", "σς"), + ("ΑΣΑ", "ασα"), + ("AΣ! AΣB", "aς! aσb"), + ("1Σ", "1σ"), + ("中Σ", "中σ"), + ("AΣ1", "aς1"), + ("AΣ中", "aς中"), + ("A\u{301}Σ", "a\u{301}ς"), + ("AΣ\u{301}", "aς\u{301}"), + ("AΣ\u{301}B", "aσ\u{301}b"), + ("\u{301}Σ", "\u{301}σ"), + ("A'Σ", "a'ς"), + ("AΣ'B", "aσ'b"), + ("AΣ\u{200d}", "aς\u{200d}"), + ("AΣ\u{200d}B", "aσ\u{200d}b"), + ("\u{345}Σ", "\u{345}σ"), + ("A\u{345}Σ", "a\u{345}ς"), + ("AΣ\u{345}", "aς\u{345}"), + ("AΣ\u{345}B", "aσ\u{345}b"), + ("𐐀Σ", "𐐨ς"), + ("AΣ𐐀", "aσ𐐨"), + ("😀Σ", "😀σ"), + ("AΣ😀", "aς😀"), + ("İΣ", "i\u{307}ς"), + ("AΣİ", "aσi\u{307}"), + ]; + for (input, expected) in cases { + let source = js_string_from_bytes(input.as_ptr(), input.len() as u32); + let result = js_string_to_lower_case(source); + assert_eq!(string_as_str(result), expected, "input: {input:?}"); + assert_eq!( + unsafe { (*result).utf16_len }, + expected.encode_utf16().count() as u32 + ); + } +} + +#[test] +fn lowercase_final_sigma_preserves_lone_surrogate_boundaries() { + for surrogate in [[0xED, 0xA0, 0x80], [0xED, 0xBF, 0xBF]] { + for (before, after, lower_before, lower_after) in [ + ("AΣ", "ΣA", "aς", "σa"), + ("A", "Σ", "a", "σ"), + ("", "AΣ", "", "aς"), + ("AΣ\u{301}", "B", "aς\u{301}", "b"), + ] { + let input = [before.as_bytes(), &surrogate, after.as_bytes()].concat(); + let expected = [lower_before.as_bytes(), &surrogate, lower_after.as_bytes()].concat(); + let source = js_string_from_wtf8_bytes(input.as_ptr(), input.len() as u32); + let result = js_string_to_lower_case(source); + let bytes = + unsafe { slice::from_raw_parts(string_data(result), (*result).byte_len as usize) }; + assert_eq!(bytes, expected); + assert_ne!( + unsafe { (*result).flags } & STRING_FLAG_HAS_LONE_SURROGATES, + 0 + ); + assert_eq!( + unsafe { (*result).utf16_len }, + compute_utf16_len_wtf8(&expected) + ); + } + } +} diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index 9e2ebac193..448520a109 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -124,6 +124,8 @@ pub(crate) mod suffix_cursor; pub(crate) mod trim_cache; mod utf16_count; +#[cfg(test)] +mod case_tests; #[cfg(test)] mod slice_tests; #[cfg(test)] diff --git a/crates/perry-runtime/src/string/slice_ops.rs b/crates/perry-runtime/src/string/slice_ops.rs index 8f6ffdc41b..921083a09a 100644 --- a/crates/perry-runtime/src/string/slice_ops.rs +++ b/crates/perry-runtime/src/string/slice_ops.rs @@ -327,14 +327,12 @@ pub extern "C" fn js_string_trim_end(s: *const StringHeader) -> *mut StringHeade /// Unicode case conversion over the raw payload (#6085). /// -/// `str::to_lowercase`/`to_uppercase` iterate `chars()`, which reads -/// continuation bytes past an exact-sized payload ending in a truncated -/// multi-byte lead. Decode with the bounded `wtf8_step` instead: sequences that -/// form a real Unicode scalar get the full `char` case mapping (identical -/// output for well-formed input, including multi-char expansions like `ß`→`SS`), -/// while a lone surrogate or a truncated/invalid sequence is copied through -/// VERBATIM — which also preserves the WTF-8 round-trip (#4793) that the old -/// `from_utf8_unchecked` path only got by accident. +/// Whole-string lowercasing handles context-dependent Greek sigma (#10116). +/// Only validated UTF-8 may enter Rust's string iterators: unchecked views +/// could read past an exact-sized payload ending in a truncated lead (#6085). +/// For WTF-8, the bounded decoder builds scalar runs for lowercasing; lone +/// surrogates and undecodable bytes are copied verbatim and end the context. +/// Uppercasing remains context-free, including expansions such as `ß`→`SS`. fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader { if !is_valid_string_ptr(s) { return js_string_from_bytes(ptr::null(), 0); @@ -365,7 +363,15 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader { return js_string_from_bytes_known_utf16(out.as_ptr(), len, len, 0); } + if !upper { + if let Ok(text) = std::str::from_utf8(bytes) { + let out = text.to_lowercase(); + return js_string_from_bytes(out.as_ptr(), out.len() as u32); + } + } + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut lowercase_run = String::new(); let mut has_lone_surrogate = false; let mut buf = [0u8; 4]; let mut i = 0usize; @@ -387,12 +393,16 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader { out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); } } else { - for c in ch.to_lowercase() { - out.extend_from_slice(c.encode_utf8(&mut buf).as_bytes()); - } + // Keep the original scalars together so Unicode's Cased + // and Case_Ignorable context is available on both sides. + lowercase_run.push(ch); } } None => { + if !lowercase_run.is_empty() { + out.extend_from_slice(lowercase_run.to_lowercase().as_bytes()); + lowercase_run.clear(); + } // Lone surrogate / truncated / stray continuation byte: copy the // raw bytes so the payload round-trips unchanged. if (0xD800..=0xDFFF).contains(&cp) { @@ -403,6 +413,9 @@ fn case_convert(s: *const StringHeader, upper: bool) -> *mut StringHeader { } i = end; } + if !lowercase_run.is_empty() { + out.extend_from_slice(lowercase_run.to_lowercase().as_bytes()); + } if has_lone_surrogate || unsafe { (*s).flags } & STRING_FLAG_HAS_LONE_SURROGATES != 0 { return js_string_from_wtf8_bytes(out.as_ptr(), out.len() as u32); } diff --git a/crates/perry-runtime/src/string/tests_guard_page.rs b/crates/perry-runtime/src/string/tests_guard_page.rs index 77d1970f8b..ac5e0976d2 100644 --- a/crates/perry-runtime/src/string/tests_guard_page.rs +++ b/crates/perry-runtime/src/string/tests_guard_page.rs @@ -453,3 +453,29 @@ fn case_convert_rejects_the_aggregate_ascii_lie() { "É (C3 89) + A + the raw truncated lead byte — NOT the input echoed back unmapped" ); } + +#[test] +fn lowercase_final_sigma_keeps_truncated_tail_bounded() { + // Every payload is twelve bytes and ends exactly at the guard page. The + // scalar run before an incomplete lead still needs final-sigma context. + for (input, expected) in [ + ( + b" ABCDE\xCE\xA3\xF0".as_slice(), + b" abcde\xCF\x82\xF0".as_slice(), + ), + ( + b" ABCD\xCE\xA3\xE2\x82".as_slice(), + b" abcd\xCF\x82\xE2\x82".as_slice(), + ), + ( + b" ABC\xCE\xA3\xF0\x9F\x98".as_slice(), + b" abc\xCF\x82\xF0\x9F\x98".as_slice(), + ), + ] { + let guarded = GuardedString::new(input); + let result = js_string_to_lower_case(guarded.ptr()); + let bytes = + unsafe { slice::from_raw_parts(string_data(result), (*result).byte_len as usize) }; + assert_eq!(bytes, expected); + } +} diff --git a/test-files/test_gap_10090_string_case_ascii_fastpath.ts b/test-files/test_gap_10090_string_case_ascii_fastpath.ts index a0bb4686f7..a469074c51 100644 --- a/test-files/test_gap_10090_string_case_ascii_fastpath.ts +++ b/test-files/test_gap_10090_string_case_ascii_fastpath.ts @@ -58,12 +58,8 @@ console.log("default-I-lower:" + "I".toLowerCase()); // "i" console.log("i-with-dot-lower:" + JSON.stringify("İ".toLowerCase())); console.log("i-with-dot-lower-units:" + JSON.stringify(units("İ".toLowerCase()))); -// NOTE: Greek final sigma (context-dependent Σ -> ς vs σ) is intentionally -// NOT covered here. It is a pre-existing gap in the untouched scalar path -// (Rust's char::to_lowercase() has no notion of the conditional Final_Sigma -// rule) unrelated to the ASCII fast path added by this file's issue, and -// asserting Node's correct output here would fail on main regardless of this -// fix. Tracked separately as #10116. +// Greek final sigma (context-dependent Σ -> ς vs σ) is covered separately +// in test_gap_10116_final_sigma.ts. // ---- Cherokee (Unicode 8.0 added case pairs) ---- console.log("cherokee-lower:" + "Ꭰ".toLowerCase()); // U+AB70 diff --git a/test-files/test_gap_10116_final_sigma.ts b/test-files/test_gap_10116_final_sigma.ts new file mode 100644 index 0000000000..379469ac98 --- /dev/null +++ b/test-files/test_gap_10116_final_sigma.ts @@ -0,0 +1,40 @@ +// #10116: context-sensitive default lowercasing, including WTF-8 boundaries. +function lower(input: string): string { return input.toLowerCase(); } +const cases = [ + ["ΑΣ", "ας"], + ["ΟΔΥΣΣΕΥΣ", "οδυσσευς"], + ["Σ", "σ"], + ["ΣΣ", "σς"], + ["ΑΣΑ", "ασα"], + ["AΣ! AΣB", "aς! aσb"], + ["1Σ", "1σ"], + ["中Σ", "中σ"], + ["AΣ1", "aς1"], + ["AΣ中", "aς中"], + ["A\u{301}Σ", "a\u{301}ς"], + ["AΣ\u{301}", "aς\u{301}"], + ["AΣ\u{301}B", "aσ\u{301}b"], + ["\u{301}Σ", "\u{301}σ"], + ["A'Σ", "a'ς"], + ["AΣ'B", "aσ'b"], + ["AΣ\u{200d}", "aς\u{200d}"], + ["AΣ\u{200d}B", "aσ\u{200d}b"], + ["\u{345}Σ", "\u{345}σ"], + ["A\u{345}Σ", "a\u{345}ς"], + ["AΣ\u{345}", "aς\u{345}"], + ["AΣ\u{345}B", "aσ\u{345}b"], + ["𐐀Σ", "𐐨ς"], + ["AΣ𐐀", "aσ𐐨"], + ["😀Σ", "😀σ"], + ["AΣ😀", "aς😀"], + ["İΣ", "i\u{307}ς"], + ["AΣİ", "aσi\u{307}"], + ["AΣ\ud800ΣA", "aς\ud800σa"], + ["A\ud800Σ", "a\ud800σ"], + ["\udfffAΣ", "\udfffaς"], + ["AΣ\u0301\udfffB", "aς\u0301\udfffb"], +]; +for (let i = 0; i < cases.length; i++) { + const actual = lower(cases[i][0]); + console.log(i, actual === cases[i][1]); +} From d8fd8bad3d9e486363dc28449284cb4132740f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 13 Sep 2026 22:56:44 +0200 Subject: [PATCH 2/2] docs: key final sigma changelog to PR 10226 --- changelog.d/{10116-final-sigma.md => 10226-final-sigma.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{10116-final-sigma.md => 10226-final-sigma.md} (100%) diff --git a/changelog.d/10116-final-sigma.md b/changelog.d/10226-final-sigma.md similarity index 100% rename from changelog.d/10116-final-sigma.md rename to changelog.d/10226-final-sigma.md