diff --git a/src/Lex.cpp b/src/Lex.cpp index 3671980..16ef99b 100644 --- a/src/Lex.cpp +++ b/src/Lex.cpp @@ -251,9 +251,12 @@ std::optional Lex::maybeLexByteString(SourceStream &S) { } bool isCharacterName(SourceStream &S, std::string &Found) { - static const std::array Names = {"space", "newline", "alarm", - "backspace", "delete", "escape", - "null", "return", "tab"}; + // Accepts both the reader's traditional names and the print names emitted by + // charReprFromCodePoint (nul/vtab/page/rubout) so printed control characters + // round-trip. "null" must precede "nul" so #\null is not truncated to #\nul. + static const std::array Names = { + "space", "newline", "alarm", "backspace", "delete", "escape", "null", + "return", "tab", "nul", "vtab", "page", "rubout"}; for (const auto *Name : Names) { if (S.isPrefix(Name)) { @@ -317,9 +320,11 @@ std::optional Lex::maybeLexSchemeChar(SourceStream &S) { (UTF8::isUTF8ReplacementCharacter(StartPtr, EndPtr) || UTF8::isGraphUTF8(StartPtr, EndPtr)) && - // followup character - (isDelimiter(S, 1) || S.peekChar(1) == EOF || - std::isspace(S.peekChar(1)))) { + // followup character: the byte right after the glyph, at offset Bytes. + // Using offset 1 would land inside a multi-byte glyph (e.g. #\λ), so its + // printed form failed to lex back as a character. + (isDelimiter(S, Bytes) || S.peekChar(Bytes) == EOF || + std::isspace(static_cast(S.peekChar(Bytes))))) { auto Subview = S.getSubviewAndSkip(Bytes); // End is the index of the last consumed char, so that Tok::size() equals // the number of bytes consumed (#\ plus the character). Using getPosition() diff --git a/test/integration/char-glyph-list-read.rkt b/test/integration/char-glyph-list-read.rkt new file mode 100644 index 0000000..2247ccf --- /dev/null +++ b/test/integration/char-glyph-list-read.rkt @@ -0,0 +1,5 @@ +;; RUN: norac %s | FileCheck %s +;; A multi-byte glyph char followed by whitespace (inside a list) must lex too; +;; the closing-delimiter case is covered by char-glyph-read.rkt. +;; CHECK: '(#\λ #\a) +(linklet () () '(#\λ #\a)) diff --git a/test/integration/char-glyph-read.rkt b/test/integration/char-glyph-read.rkt new file mode 100644 index 0000000..9298b7f --- /dev/null +++ b/test/integration/char-glyph-read.rkt @@ -0,0 +1,4 @@ +;; RUN: norac %s | FileCheck %s +;; Reading a printed multi-byte glyph char literal back must yield the same char. +;; CHECK: #\λ +(linklet () () '#\λ) diff --git a/test/integration/char-nul-read.rkt b/test/integration/char-nul-read.rkt new file mode 100644 index 0000000..563a963 --- /dev/null +++ b/test/integration/char-nul-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\nul +(linklet () () '#\nul) diff --git a/test/integration/char-null-read.rkt b/test/integration/char-null-read.rkt new file mode 100644 index 0000000..9e91484 --- /dev/null +++ b/test/integration/char-null-read.rkt @@ -0,0 +1,4 @@ +;; RUN: norac %s | FileCheck %s +;; The traditional #\null name must still lex (not be truncated to #\nul). +;; CHECK: #\null +(linklet () () '#\null) diff --git a/test/integration/char-page-read.rkt b/test/integration/char-page-read.rkt new file mode 100644 index 0000000..d5813b0 --- /dev/null +++ b/test/integration/char-page-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\page +(linklet () () '#\page) diff --git a/test/integration/char-rubout-read.rkt b/test/integration/char-rubout-read.rkt new file mode 100644 index 0000000..f346405 --- /dev/null +++ b/test/integration/char-rubout-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\rubout +(linklet () () '#\rubout) diff --git a/test/integration/char-vtab-read.rkt b/test/integration/char-vtab-read.rkt new file mode 100644 index 0000000..dc77ea4 --- /dev/null +++ b/test/integration/char-vtab-read.rkt @@ -0,0 +1,3 @@ +;; RUN: norac %s | FileCheck %s +;; CHECK: #\vtab +(linklet () () '#\vtab)