From 089825c5c29f973da88550d12b50d1368af89ad0 Mon Sep 17 00:00:00 2001 From: Paulo Matos Date: Wed, 1 Jul 2026 19:01:56 +0200 Subject: [PATCH 1/2] Fix character print/read round-trip for glyphs and control names (#73) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit maybeLexSchemeChar checked the followup delimiter at byte offset 1, which falls inside a multi-byte UTF-8 glyph, so printed forms like #\λ failed to lex back as a character (and drove parseQuote/parseValue into unbounded recursion). Advance the followup offset by the glyph's UTF-8 byte length. Also extend the reader's character-name table with the print names nul/vtab/page/rubout so decoded control characters round-trip. --- src/Lex.cpp | 17 +++++++++++------ test/integration/char-glyph-read.rkt | 4 ++++ test/integration/char-nul-read.rkt | 3 +++ test/integration/char-null-read.rkt | 4 ++++ test/integration/char-page-read.rkt | 3 +++ test/integration/char-rubout-read.rkt | 3 +++ test/integration/char-vtab-read.rkt | 3 +++ 7 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 test/integration/char-glyph-read.rkt create mode 100644 test/integration/char-nul-read.rkt create mode 100644 test/integration/char-null-read.rkt create mode 100644 test/integration/char-page-read.rkt create mode 100644 test/integration/char-rubout-read.rkt create mode 100644 test/integration/char-vtab-read.rkt diff --git a/src/Lex.cpp b/src/Lex.cpp index 3671980..16ef99b 100644 --- a/src/Lex.cpp +++ b/src/Lex.cpp @@ -251,9 +251,12 @@ std::optional Lex::maybeLexByteString(SourceStream &S) { } bool isCharacterName(SourceStream &S, std::string &Found) { - static const std::array Names = {"space", "newline", "alarm", - "backspace", "delete", "escape", - "null", "return", "tab"}; + // Accepts both the reader's traditional names and the print names emitted by + // charReprFromCodePoint (nul/vtab/page/rubout) so printed control characters + // round-trip. "null" must precede "nul" so #\null is not truncated to #\nul. + static const std::array Names = { + "space", "newline", "alarm", "backspace", "delete", "escape", "null", + "return", "tab", "nul", "vtab", "page", "rubout"}; for (const auto *Name : Names) { if (S.isPrefix(Name)) { @@ -317,9 +320,11 @@ std::optional Lex::maybeLexSchemeChar(SourceStream &S) { (UTF8::isUTF8ReplacementCharacter(StartPtr, EndPtr) || UTF8::isGraphUTF8(StartPtr, EndPtr)) && - // followup character - (isDelimiter(S, 1) || S.peekChar(1) == EOF || - std::isspace(S.peekChar(1)))) { + // followup character: the byte right after the glyph, at offset Bytes. + // Using offset 1 would land inside a multi-byte glyph (e.g. #\λ), so its + // printed form failed to lex back as a character. + (isDelimiter(S, Bytes) || S.peekChar(Bytes) == EOF || + std::isspace(static_cast(S.peekChar(Bytes))))) { auto Subview = S.getSubviewAndSkip(Bytes); // End is the index of the last consumed char, so that Tok::size() equals // the number of bytes consumed (#\ plus the character). Using getPosition() diff --git a/test/integration/char-glyph-read.rkt b/test/integration/char-glyph-read.rkt new file mode 100644 index 0000000..9298b7f --- /dev/null +++ b/test/integration/char-glyph-read.rkt @@ -0,0 +1,4 @@ +;; RUN: norac %s | FileCheck %s +;; Reading a printed multi-byte glyph char literal back must yield the same char. +;; CHECK: #\λ +(linklet () () '#\λ) diff --git a/test/integration/char-nul-read.rkt b/test/integration/char-nul-read.rkt new file mode 100644 index 0000000..563a963 --- /dev/null +++ b/test/integration/char-nul-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\nul +(linklet () () '#\nul) diff --git a/test/integration/char-null-read.rkt b/test/integration/char-null-read.rkt new file mode 100644 index 0000000..9e91484 --- /dev/null +++ b/test/integration/char-null-read.rkt @@ -0,0 +1,4 @@ +;; RUN: norac %s | FileCheck %s +;; The traditional #\null name must still lex (not be truncated to #\nul). +;; CHECK: #\null +(linklet () () '#\null) diff --git a/test/integration/char-page-read.rkt b/test/integration/char-page-read.rkt new file mode 100644 index 0000000..d5813b0 --- /dev/null +++ b/test/integration/char-page-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\page +(linklet () () '#\page) diff --git a/test/integration/char-rubout-read.rkt b/test/integration/char-rubout-read.rkt new file mode 100644 index 0000000..f346405 --- /dev/null +++ b/test/integration/char-rubout-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\rubout +(linklet () () '#\rubout) diff --git a/test/integration/char-vtab-read.rkt b/test/integration/char-vtab-read.rkt new file mode 100644 index 0000000..dc77ea4 --- /dev/null +++ b/test/integration/char-vtab-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\vtab +(linklet () () '#\vtab) From 02d360380deb1dcd6b625cb58e2d4ad6b1f8b434 Mon Sep 17 00:00:00 2001 From: Paulo Matos Date: Wed, 1 Jul 2026 19:42:42 +0200 Subject: [PATCH 2/2] Test glyph char followed by whitespace inside a list (#73) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing glyph test puts a delimiter right after #\λ, so the whitespace branch of maybeLexSchemeChar's followup check was never exercised. A glyph char in a list ('(#\λ #\a)) covers it. --- test/integration/char-glyph-list-read.rkt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 test/integration/char-glyph-list-read.rkt diff --git a/test/integration/char-glyph-list-read.rkt b/test/integration/char-glyph-list-read.rkt new file mode 100644 index 0000000..2247ccf --- /dev/null +++ b/test/integration/char-glyph-list-read.rkt @@ -0,0 +1,5 @@ +;; RUN: norac %s | FileCheck %s +;; A multi-byte glyph char followed by whitespace (inside a list) must lex too; +;; the closing-delimiter case is covered by char-glyph-read.rkt. +;; CHECK: '(#\λ #\a) +(linklet () () '(#\λ #\a))