From fa1892d930140393f7c8e05e15a12f458189afc0 Mon Sep 17 00:00:00 2001 From: takeokunn Date: Mon, 3 Aug 2026 20:10:18 +0900 Subject: [PATCH] fix(syntax): implement LFE's hash-dispatch forms and quoted symbols Four reader defects, three of them silent. The headline is not the parser: **the pre-change formatter silently corrupted 283 of 2600 LFE files at exit 0**, rewriting `#M()` as `#M ()`, which LFE cannot scan at all (`{illegal_token,"#M"}`). Round-trip idempotence passed on the broken binary -- 0 refusals, 0 non-reparsing outputs, 0 non-idempotent. Only LFE's own token stream as an external oracle caught it: LFE reads the formatted output as the same program pre 2317 / 2600 post 2674 / 2674 pre 283 DIFFERENT post 0 The defects, measured over 2735 .lfe files from 236 repos: - `#B(` / `#M(` / `#S(` orphaned from their list, inflating arity at exit 0. `#M(` is 336 files (12.3%); `#B(` is only 27. The brief's combined figure was the right order but the wrong composition. - **`#"..."` binary strings, not in the brief at all, and the biggest one** -- 307 files (11.2%), 139 with whitespace or a delimiter inside. `#"text/plain; version=0.0.4"` split into three atoms; `#" a"` desynchronised the rest of the file into giant garbage atoms; `#"a)b"` closed its enclosing list early. - `#\c` character literals -- 84 files, not the 1 reported. - `|quoted atoms|` -- 44 files, only 29 actually splitting. The brief's 95 was ~3x too high. LFE 2.2.0 has **no leex file**; `src/lfe_scan.erl` is hand-written. The complete `#`-dispatch set is read from `scan_hash1`/`scan_hash2`. Three findings that shaped the code: - `#B(` is its own token, not a prefix -- `spell1` reads `sexpr -> '#B(' proper_list ')'` -- but it is structurally identical to `#(`, so it is modelled the same way. - `#S(` is scanned and then **unparseable**: `spell1` declares the terminal but has no production using it, and LFE answers `{illegal,'#S('}`. Lexed anyway, because orphaning `#S` in an already-broken file is worse. - `|` opens a quoted symbol only at a token's first byte (`start_symbol_char($|)` is false but `symbol_char` falls through to the printable range), so `a|b c|d` is two symbols. Reusing the CL "anywhere" rule would have fused them; hence `BarQuoting::{None, Anywhere,TokenStart}`. `$c` is **not** LFE -- `$` is an ordinary symbol char -- so the `$(` concern in the brief does not apply. `#B(` deliberately does not reuse `HashLiteral`, whose `as_source()` is `"#"`: that would have made `inline-function`'s renderer turn a binary into a tuple. New silent corruption in place of old. Differential over the 2639 files both builds parse, aligned by byte span rather than position: 2667 list forms in 394 files changed child count, 3241 phantom nodes gone. Nine forms *gained* children and all nine were inspected -- each correct, e.g. a `#"..."` that had been closing its `defun` early. Parse failures 96 -> 22, with 74 newly parsing and 0 newly failing; 6 of the 22 are files LFE's own parser also rejects. Against LFE's scanner directly: 2113/2549 -> 2540/2549. Differential fuzz 670/2500 -> 2500/2500, with zero "both reject" cases, so no agreement is coincidental. Other dialects: **76164 parses, zero differences** across CL, Elisp, Scheme, Racket, Clojure, Fennel, Janet, Hy, Carp and the legacy reader. A test pins `bar_quoting` and `character_literal_is_exactly_one_char` per dialect against `Dialect::ALL.len()`, so adding a dialect forces a decision. `lfe-catch-swallows-exit` findings are unmoved (1542 = 1542, zero pre-only, zero post-only). The +41 in the whole-corpus run comes entirely from the 74 files that previously could not be linted at all. No golden or pinned count moved, because the repo's LFE fixture contains none of these constructs -- that fixture is thin, and left alone rather than moving several pinned counts for coverage the unit tests give precisely. Deferred and reported: triple-quoted strings (`scan_tq_string`, LFE 2.2) -- 1 file; `"` not being an atom boundary, so `%"%"` is one atom where LFE sees two forms -- cross-dialect, 2 files; improper-list `.` read as an ordinary atom child -- 140 files. One normalisation documented on the enum rather than left to be discovered: `as_source()` spells these upper case (`#B`), matching LFE's own printer, so inlining a macro body containing `#b(...)` returns `#B(...)`. Same literal to LFE's reader. --- packages/core/syntax/src/sexpr/parser.rs | 41 ++- packages/core/syntax/src/sexpr/reader.rs | 8 + .../core/syntax/src/sexpr/reader_policy.rs | 154 +++++++- .../core/syntax/src/sexpr/tests/parser.rs | 336 ++++++++++++++++++ packages/core/syntax/src/sexpr/tree.rs | 43 ++- .../domain/macro_expansion/mod.rs | 9 + 6 files changed, 569 insertions(+), 22 deletions(-) diff --git a/packages/core/syntax/src/sexpr/parser.rs b/packages/core/syntax/src/sexpr/parser.rs index 0b95babe..7337eb41 100644 --- a/packages/core/syntax/src/sexpr/parser.rs +++ b/packages/core/syntax/src/sexpr/parser.rs @@ -2,7 +2,7 @@ use thiserror::Error; use crate::dialect::Dialect; -use super::reader_policy::{DialectReaderPolicy, LongStringExtent, ReaderMacro}; +use super::reader_policy::{BarQuoting, DialectReaderPolicy, LongStringExtent, ReaderMacro}; use super::tree::{Comment, Node, NodeKind, ReaderPrefix, ReaderPrefixes, SyntaxTree}; use super::types::{ByteOffset, ByteSpan, Delimiter, NodeId}; @@ -448,16 +448,31 @@ impl<'a> Parser<'a> { } fn consume_atom_body(&mut self) -> std::result::Result<(), ParseError> { - self.consume_character_literal(); + let token_start = self.pos.get(); + if self.consume_character_literal() { + return Ok(()); + } while self.pos.get() < self.bytes.len() { let byte = self.current_byte(); if self.policy.supports_single_escape() && byte == b'\\' { self.consume_single_escape()?; continue; } - if self.policy.supports_bar_quoted_symbols() && byte == b'|' { - self.consume_multiple_escape()?; - continue; + if byte == b'|' { + match self.policy.bar_quoting() { + BarQuoting::Anywhere => { + self.consume_multiple_escape()?; + continue; + } + // LFE opens a quoted symbol only at a token's first byte; + // anywhere else the `|` is an ordinary constituent, so + // `a|b c|d` stays two symbols rather than becoming one. + BarQuoting::TokenStart if self.pos.get() == token_start => { + self.consume_multiple_escape()?; + continue; + } + BarQuoting::TokenStart | BarQuoting::None => {} + } } if self.policy.is_atom_boundary(self.bytes, self.pos.get()) { break; @@ -467,18 +482,28 @@ impl<'a> Parser<'a> { Ok(()) } - fn consume_character_literal(&mut self) { + /// Consumes a character literal at the cursor, if one starts there. + /// + /// Returns whether the literal is *complete* — whether the token ends where + /// the literal does, so no further scanning may happen. Only LFE says yes: + /// its literal is the two-byte `#\` and exactly one character, with no + /// named forms and no escapes, so `#\"` ends at the quote and the string + /// that follows is a separate token. Elsewhere the name may be several + /// characters (`#\space`, `?\C-x`, `\newline`) and the caller must keep + /// scanning to the next boundary. + fn consume_character_literal(&mut self) -> bool { let Some(prefix_width) = self .policy .character_literal_prefix_width(self.bytes, self.pos.get()) else { - return; + return false; }; self.advance_by(prefix_width); let Some(character) = self.input[self.pos.get()..].chars().next() else { - return; + return false; }; self.advance_by(character.len_utf8()); + self.policy.character_literal_is_exactly_one_char() } /// Consumes a Lisp single-escape (`\`) and the following character literally. diff --git a/packages/core/syntax/src/sexpr/reader.rs b/packages/core/syntax/src/sexpr/reader.rs index bee8a8b3..dd6046b4 100644 --- a/packages/core/syntax/src/sexpr/reader.rs +++ b/packages/core/syntax/src/sexpr/reader.rs @@ -46,7 +46,15 @@ pub fn apply_reader_prefix_context( // anonymous functions and metadata targets), so treat them like // `Function` rather than opaque data: keep traversing normally // instead of hiding the contents from rename/reference tracking. + // LFE's `#B(…)`, `#M(…)` and `#S(…)` sit here for the same reason: + // their elements are ordinary expressions, not opaque data. A + // binary segment is `(value (size N) (unit 8))` and a map literal's + // values are arbitrary forms, so rename and reference tracking must + // keep descending into them. ReaderPrefix::HashLiteral + | ReaderPrefix::LfeBinary + | ReaderPrefix::LfeMap + | ReaderPrefix::LfeStruct | ReaderPrefix::Metadata | ReaderPrefix::ReaderConditional | ReaderPrefix::ReaderConditionalSplicing => {} diff --git a/packages/core/syntax/src/sexpr/reader_policy.rs b/packages/core/syntax/src/sexpr/reader_policy.rs index 5bc62004..f1a5abe4 100644 --- a/packages/core/syntax/src/sexpr/reader_policy.rs +++ b/packages/core/syntax/src/sexpr/reader_policy.rs @@ -21,6 +21,18 @@ pub(super) enum ReaderMacro { }, } +/// Where a `|...|` region may begin inside a token. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum BarQuoting { + /// A `|` is an ordinary symbol constituent wherever it appears. + None, + /// CLHS 2.1.4.2 / R7RS 2.1: a multiple-escape may open anywhere in a token. + Anywhere, + /// LFE: a `|` opens a quoted symbol only where a token starts; anywhere + /// else it is an ordinary constituent. + TokenStart, +} + /// How far a Janet long string reaches from its opening backtick run. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum LongStringExtent { @@ -105,15 +117,31 @@ impl DialectReaderPolicy { ) } - /// Whether `|...|` reads as one symbol rather than a token boundary. + /// Where `|...|` reads as one symbol rather than a token boundary. /// /// R7RS 2.1 gives Scheme the same vertical-line notation Common Lisp has - /// in CLHS 2.1.4.2, so `|Foo Bar|` is a single identifier in both. - pub(super) const fn supports_bar_quoted_symbols(self) -> bool { - matches!( - self.dialect, - Dialect::CommonLisp | Dialect::Scheme | Dialect::Racket | Dialect::Unknown - ) + /// in CLHS 2.1.4.2, so `|Foo Bar|` is a single identifier in both, and in + /// both a `|` may open a quoted region part-way through a token. + /// + /// LFE has the notation but not that second property, and the difference + /// is explicit in `lfe_scan.erl`: `start_symbol_char($|) -> false` sends a + /// leading `|` to `scan_qsymbol`, while `symbol_char/1` has no `$|` clause + /// at all, so it falls through to `(C > $\s) and (C =< $~)` — true for + /// `|` (124). A `|` inside a token is therefore an ordinary constituent + /// and `a|b c|d` is the two symbols `a|b` and `c|d`, not one. + pub(super) const fn bar_quoting(self) -> BarQuoting { + match self.dialect { + Dialect::CommonLisp | Dialect::Scheme | Dialect::Racket | Dialect::Unknown => { + BarQuoting::Anywhere + } + Dialect::Lfe => BarQuoting::TokenStart, + Dialect::EmacsLisp + | Dialect::Clojure + | Dialect::Hy + | Dialect::Carp + | Dialect::Janet + | Dialect::Fennel => BarQuoting::None, + } } /// Whether a bare `\` escapes the next character *outside* `|...|`. @@ -231,7 +259,11 @@ impl DialectReaderPolicy { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); let width = match self.dialect { - Dialect::Scheme | Dialect::Racket if byte == b'#' && next == Some(b'\\') => 2, + Dialect::Scheme | Dialect::Racket | Dialect::Lfe + if byte == b'#' && next == Some(b'\\') => + { + 2 + } Dialect::Clojure if byte == b'\\' => 1, Dialect::EmacsLisp if byte == b'?' && next == Some(b'\\') => 2, Dialect::EmacsLisp if byte == b'?' => 1, @@ -240,15 +272,40 @@ impl DialectReaderPolicy { bytes.get(pos + width).is_some().then_some(width) } + /// Whether a character literal is *exactly* its prefix plus one character, + /// so the token ends there rather than running on to the next boundary. + /// + /// LFE is the only dialect here where it is. `lfe_scan.erl` spells the + /// whole grammar in one clause: + /// + /// ```erlang + /// scan_hash2([$\\,C|Cs], Line, Col, [], St) -> + /// {ok,{number,Line,C},Cs,Line,Col+2,St}; + /// ``` + /// + /// One character, taken verbatim. There are no named characters, and no + /// escape processing at all — `#\n` is the letter `n` (110), not a newline. + /// So `#\"abc"` is the character `"` followed by the string `abc`, and + /// running the token on to the next boundary instead would swallow the + /// string's opening quote and glue the rest of the file into one atom. + /// + /// Everywhere else the name may be longer than one character — Scheme's + /// `#\space`, Emacs Lisp's `?\C-x`, Clojure's `\newline` — so the token has + /// to keep scanning, and nothing below may change for them. + pub(super) const fn character_literal_is_exactly_one_char(self) -> bool { + matches!(self.dialect, Dialect::Lfe) + } + pub(super) fn classify_reader_macro(self, bytes: &[u8], pos: usize) -> Option { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); let third = bytes.get(pos + 2).copied(); match self.dialect { - Dialect::Unknown | Dialect::Lfe | Dialect::Hy | Dialect::Carp => { + Dialect::Unknown | Dialect::Hy | Dialect::Carp => { self.classify_legacy(byte, next, third) } + Dialect::Lfe => self.classify_lfe(bytes, pos), Dialect::CommonLisp => self.classify_common_lisp(bytes, pos), Dialect::EmacsLisp => self.classify_emacs_lisp(bytes, pos), Dialect::Scheme | Dialect::Racket => self.classify_scheme(bytes, pos), @@ -300,6 +357,85 @@ impl DialectReaderPolicy { classify_shared_prefix(byte, next, third) } + /// LFE's `#`-dispatch table, from `lfe_scan.erl`'s `scan_hash1`/`scan_hash2`. + /// + /// That is the complete set, in the scanner's own order. `scan_hash` + /// collects decimal digits first (`scan_hash_digits`), and every form + /// except `#r` requires that digit run to be empty: + /// + /// | source | token | handled by | + /// |---------------|---------------------------|-------------------------------| + /// | `#(` | `'#('` tuple open | [`classify_shared_prefix`] | + /// | `#B(` `#b(` | `'#B('` binary open | this arm | + /// | `#M(` `#m(` | `'#M('` map open | this arm | + /// | `#S(` `#s(` | `'#S('` struct open | this arm | + /// | `#"…"` | `binary` string | this arm | + /// | `#\C` | `{number,_,C}` one char | `character_literal_prefix_width` | + /// | `#'f/2` | `'#\''` fun reference | [`classify_shared_prefix`] | + /// | `#.` | `'#.'` read-eval | [`classify_shared_prefix`] | + /// | `#\|` | block comment | `supports_block_comments` | + /// | `#*1010` | base-2 number | scans as a plain atom | + /// | `#b…` `#o…` `#d…` `#x…` | based numbers | scan as plain atoms | + /// | `#r…` | base-2..36 number | scans as a plain atom | + /// | `` #` `` `#;` `#,` `#,@` | scanned, no grammar production | left as they were | + /// + /// The based-number forms need nothing: `#x1f` has no delimiter in it, so + /// the ordinary atom scanner already takes the whole token. The last row is + /// deliberately untouched — `lfe_parse.spell1` declares no production for + /// those tokens, so LFE itself refuses them, and the existing readings are + /// neither more nor less wrong than they were. + /// + /// Everything else falls through to [`Self::classify_legacy`], which is + /// what LFE used before this function existed, so no reading changes except + /// the ones named above. + fn classify_lfe(self, bytes: &[u8], pos: usize) -> Option { + let byte = *bytes.get(pos)?; + let next = bytes.get(pos + 1).copied(); + let third = bytes.get(pos + 2).copied(); + + if byte != b'#' { + return self.classify_legacy(byte, next, third); + } + + // `#B(`, `#M(`, `#S(` are single opening tokens in `scan_hash2`, each + // closed by a plain `)` in `lfe_parse.spell1`. Reading the two-byte + // dispatch as a prefix on the list that follows gives exactly that + // shape, and is how `#(` has always been read. Without it the `#B` + // scanned as its own atom and the list became a *sibling*, so + // `(f #B(1 2) X)` had four children where LFE sees three -- silently, + // at exit 0, which is why this is the defect that matters most. + if third == Some(b'(') { + match next { + Some(b'b' | b'B') => return prefix(ReaderPrefix::LfeBinary, 2), + Some(b'm' | b'M') => return prefix(ReaderPrefix::LfeMap, 2), + Some(b's' | b'S') => return prefix(ReaderPrefix::LfeStruct, 2), + _ => {} + } + } + + // `#"…"` is one `binary` token (`scan_hash1([$"|Cs], …)` hands + // straight to `scan_binary_string`). Treating the `#` as a prefix on + // the string that follows keeps the whole literal in one node, where + // before the `#` glued onto the string's *first word* and every later + // word became a sibling atom: `#"text/plain; version=0.0.4"` split + // into three, and one containing a `)` closed its enclosing list early. + if next == Some(b'"') { + return prefix(ReaderPrefix::HashLiteral, 1); + } + + // `#\` with nothing after it is a truncated character literal. + // Refusing it is what stops the formatter's trailing newline from + // becoming the literal's character on the next parse, which would make + // `format(format(x))` differ from `format(x)`. Scheme and Racket + // already refuse the same input for the same reason; LFE's own scanner + // refuses it too, as `{illegal_token,"#\\"}`. + if next == Some(b'\\') && third.is_none() { + return Some(ReaderMacro::UnsupportedDispatch { width: 1 }); + } + + self.classify_legacy(byte, next, third) + } + fn classify_common_lisp(self, bytes: &[u8], pos: usize) -> Option { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); diff --git a/packages/core/syntax/src/sexpr/tests/parser.rs b/packages/core/syntax/src/sexpr/tests/parser.rs index fa001fc9..22ddb4c0 100644 --- a/packages/core/syntax/src/sexpr/tests/parser.rs +++ b/packages/core/syntax/src/sexpr/tests/parser.rs @@ -1681,3 +1681,339 @@ fn clojure_namespaced_map_width_stops_at_the_namespace() { // Comments are not whitespace to Clojure's reader here. assert_eq!(width("#:foo ;; c\n{:a 1}"), None); } + +// --------------------------------------------------------------------------- +// LFE reader +// +// The reference is LFE 2.2.0's own scanner, `src/lfe_scan.erl`, and its +// grammar, `src/lfe_parse.spell1`. Every expected reading below is that +// scanner's token stream, checked by running the case through +// `lfe_scan:string/1` -- not a guess at what the notation ought to mean. +// --------------------------------------------------------------------------- + +/// The source text of every child of the document's single top-level list. +fn lfe_children(input: &'static str) -> Vec<&'static str> { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect() +} + +/// `#B(`, `#M(` and `#S(` are single *opening* tokens in `scan_hash2`: +/// +/// ```erlang +/// scan_hash2([C,$\(|Cs], Line, Col, [], St) when (C =:= $b) or (C =:= $B) -> +/// {ok,{'#B(',Line},Cs,Line,Col,St}; +/// ``` +/// +/// closed by a plain `)`, since `lfe_parse.spell1` reads +/// `sexpr -> '#B(' proper_list ')'`. Taking the two-byte dispatch as a prefix +/// on the following list gives exactly that shape. +/// +/// Before this, the `#B` scanned as its own atom and the list became its +/// *sibling*, so `(f #B(1 2) X)` had four children where LFE sees three -- +/// silently, at exit 0, which made every arity-sensitive rule read the form +/// wrong. +#[test] +fn lfe_hash_letter_collections_stay_attached_to_their_list() { + struct Case { + input: &'static str, + children: &'static [&'static str], + prefix: ReaderPrefix, + } + + let cases = [ + Case { + input: "(f #B(1 2) X)", + children: &["f", "#B(1 2)", "X"], + prefix: ReaderPrefix::LfeBinary, + }, + // The scanner accepts either case, and the document keeps its own. + Case { + input: "(f #b(1 2) X)", + children: &["f", "#b(1 2)", "X"], + prefix: ReaderPrefix::LfeBinary, + }, + Case { + input: "(g #M(a 1 b 2) Y)", + children: &["g", "#M(a 1 b 2)", "Y"], + prefix: ReaderPrefix::LfeMap, + }, + Case { + input: "(g #m(a 1) Y)", + children: &["g", "#m(a 1)", "Y"], + prefix: ReaderPrefix::LfeMap, + }, + // `lfe_scan` emits `'#S('` and `lfe_parse.spell1` declares it a + // terminal, but 2.2.0 has no production using it, so LFE itself + // answers `{illegal,'#S('}`. Lexing it anyway beats orphaning the `#S` + // from its list in a file that is already broken. + Case { + input: "(h #S(point x 1) Z)", + children: &["h", "#S(point x 1)", "Z"], + prefix: ReaderPrefix::LfeStruct, + }, + // The tuple opener, which has always worked, pinned so the new arms + // cannot displace it. + Case { + input: "(i #(1 2) W)", + children: &["i", "#(1 2)", "W"], + prefix: ReaderPrefix::HashLiteral, + }, + ]; + + for case in cases { + assert_eq!(lfe_children(case.input), case.children, "{}", case.input); + let tree = SyntaxTree::parse_with_dialect(case.input, Dialect::Lfe).expect("valid"); + let literal = &tree.root_view().children[0].children[1]; + assert_eq!(literal.kind, ExpressionKind::List, "{}", case.input); + assert_eq!(literal.reader_prefixes, vec![case.prefix], "{}", case.input); + } +} + +/// `#b` and its friends only open a collection when a `(` follows. +/// `scan_hash2` orders the binary-token clause before the based-number one for +/// exactly this reason ("Scan binary tokens, these must come before the based +/// number"), so `#b1010` stays the number ten. +#[test] +fn lfe_based_numbers_are_not_collection_openers() { + for input in [ + "(f #b1010 x)", + "(f #B1010 x)", + "(f #x1f x)", + "(f #o17 x)", + "(f #d99 x)", + "(f #2r1010 x)", + "(f #*1010 x)", + ] { + let children = lfe_children(input); + assert_eq!(children.len(), 3, "{input}"); + assert_eq!(children[0], "f", "{input}"); + assert_eq!(children[2], "x", "{input}"); + } +} + +/// `scan_hash1([$"|Cs], Line, Col, [], St) -> scan_binary_string(...)` makes +/// `#"..."` a single `binary` token. +/// +/// Before this the `#` glued onto the string's *first word* and every later +/// word became a sibling atom, so `#"text/plain; version=0.0.4"` split into +/// three and one containing a `)` closed its enclosing list early. Binary +/// strings are the most common of these constructs in real LFE. +#[test] +fn lfe_binary_strings_are_one_atom() { + let cases: &[(&str, &[&str])] = &[ + ("(f #\"GET\" x)", &["f", "#\"GET\"", "x"]), + ("(f #\"a b\" x)", &["f", "#\"a b\"", "x"]), + ("(f #\"x)y\" x)", &["f", "#\"x)y\"", "x"]), + ("(f #\"\" x)", &["f", "#\"\"", "x"]), + ( + "(f #\"text/plain; version=0.0.4\" x)", + &["f", "#\"text/plain; version=0.0.4\"", "x"], + ), + ("(f #\"esc \\\" q\" x)", &["f", "#\"esc \\\" q\"", "x"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// LFE's whole character-literal grammar is one clause: +/// +/// ```erlang +/// scan_hash2([$\\,C|Cs], Line, Col, [], St) -> +/// {ok,{number,Line,C},Cs,Line,Col+2,St}; +/// ``` +/// +/// Two bytes of prefix and exactly one character, taken verbatim. There are no +/// named characters and no escape processing at all, so `#\newline` is the +/// letter `n` followed by the symbol `ewline` -- `lfe_scan:string/1` answers +/// `[{number,1,110},{symbol,1,ewline}]` for it. +#[test] +fn lfe_character_literal_is_exactly_one_character() { + let cases: &[(&str, &[&str])] = &[ + // The delimiters. These used to restructure the tree outright: `#\(` + // scanned as the atom `#\` and then *opened a list*. + ("(list #\\( #\\))", &["list", "#\\(", "#\\)"]), + // A `;` would otherwise start a comment and eat the rest of the line. + ("(list #\\; a)", &["list", "#\\;", "a"]), + // A `"` would otherwise open a string and swallow the file. + ("(list #\\\" a)", &["list", "#\\\"", "a"]), + ("(list #\\\\ a)", &["list", "#\\\\", "a"]), + ("(list #\\| a)", &["list", "#\\|", "a"]), + ("(list #\\a #\\b)", &["list", "#\\a", "#\\b"]), + // One character means one character: the rest is a separate symbol. + ("(f #\\newline)", &["f", "#\\n", "ewline"]), + // A multi-byte character is one character, not one byte. + ("(f #\\\u{e9} x)", &["f", "#\\\u{e9}", "x"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// `start_symbol_char($|) -> false` sends a *leading* `|` to `scan_qsymbol`, +/// which runs to the closing `|` taking `\C` verbatim; whitespace and +/// delimiters inside are ordinary content. But `symbol_char/1` has no `$|` +/// clause, so it falls through to `(C > $\s) and (C =< $~)` -- true for `|` +/// (124) -- and a `|` *inside* a token is an ordinary constituent. +/// +/// Both halves matter. Without the first, `'|foo bar|` split at the space; +/// with the first but not the second, `a|b c|d` would fuse into one symbol. +#[test] +fn lfe_bar_quoted_symbols_open_only_at_a_token_start() { + let cases: &[(&str, &[&str])] = &[ + ("(f '|foo bar|)", &["f", "'|foo bar|"]), + ("(f |a(b| x)", &["f", "|a(b|", "x"]), + ("(f |x\\|y| x)", &["f", "|x\\|y|", "x"]), + ("(f || x)", &["f", "||", "x"]), + ("(f |;| x)", &["f", "|;|", "x"]), + // A newline inside is content: `scan_qsymbol1` has an explicit `$\n` + // clause that keeps collecting. + ("(f |multi\nline| x)", &["f", "|multi\nline|", "x"]), + // Mid-token, an ordinary constituent. Two symbols, not one. + ("(f a|b c|d)", &["f", "a|b", "c|d"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// An unterminated `|` fails loudly rather than consuming the rest of the file +/// as one symbol. +/// +/// LFE refuses it too -- `scan_qsymbol1(eof, ...)` raises +/// `{illegal_chars,[$| | Symcs]}` -- so this agrees with the reference reader +/// rather than inventing a rule. Reading to EOF as one atom would be exactly +/// the silent corruption this work exists to remove. +#[test] +fn lfe_unterminated_bar_quoted_symbol_is_refused() { + let error = SyntaxTree::parse_with_dialect("(f |unterminated\n(g 1)\n", Dialect::Lfe) + .expect_err("unterminated multiple escape"); + assert_eq!(error, ParseError::UnterminatedSymbol(3)); +} + +/// `#\` with nothing after it is a truncated literal, not the character for +/// nothing. LFE answers `{illegal_token,"#\\"}`. +/// +/// Accepting it as a complete atom would make the formatter non-idempotent: +/// it appends a trailing newline, the truncated literal claims it as its +/// character, and the next pass appends another. Scheme and Racket already +/// refuse the same input for the same reason. +#[test] +fn lfe_truncated_character_literal_is_refused() { + let error = SyntaxTree::parse_with_dialect("(f #\\", Dialect::Lfe) + .expect_err("truncated character literal"); + assert_eq!( + error, + ParseError::UnsupportedReaderDispatch { + dispatch: "#".to_owned(), + position: 3, + } + ); +} + +/// Everything above is LFE-only. +/// +/// `#(` is a vector in Scheme and Racket and a set or lambda in Clojure, so +/// these bytes are live elsewhere with other meanings; a letter between the +/// `#` and the `(` means something different or nothing in all ten. This pins +/// them against a later edit widening `classify_lfe`'s arms by accident. +#[test] +fn lfe_hash_letter_collections_do_not_leak_into_other_dialects() { + for dialect in [ + Dialect::CommonLisp, + Dialect::EmacsLisp, + Dialect::Scheme, + Dialect::Racket, + Dialect::Clojure, + Dialect::Fennel, + Dialect::Janet, + Dialect::Hy, + Dialect::Carp, + Dialect::Unknown, + ] { + let label = dialect.label(); + // Refusing the input is fine; what must not happen is reading it as + // one prefixed list the way LFE now does. + let Ok(tree) = SyntaxTree::parse_with_dialect("(f #B(1 2) X)", dialect) else { + continue; + }; + let prefixes: Vec = tree.root_view().children[0] + .children + .iter() + .flat_map(|child| child.reader_prefixes.clone()) + .collect(); + assert!( + !prefixes.contains(&ReaderPrefix::LfeBinary), + "{label} produced an LFE binary literal" + ); + } +} + +/// The two lexical switches this change added, pinned per dialect. +/// +/// The table is worth more than the prose: `bar_quoting` replaced a boolean, +/// and its mapping has to stay exactly what that boolean was for the ten +/// dialects LFE is not. +#[test] +fn lfe_lexical_switches_are_scoped_to_lfe() { + use crate::sexpr::reader_policy::BarQuoting; + + let expected = [ + (Dialect::CommonLisp, BarQuoting::Anywhere, false), + (Dialect::EmacsLisp, BarQuoting::None, false), + (Dialect::Lfe, BarQuoting::TokenStart, true), + (Dialect::Scheme, BarQuoting::Anywhere, false), + (Dialect::Racket, BarQuoting::Anywhere, false), + (Dialect::Clojure, BarQuoting::None, false), + (Dialect::Hy, BarQuoting::None, false), + (Dialect::Carp, BarQuoting::None, false), + (Dialect::Janet, BarQuoting::None, false), + (Dialect::Fennel, BarQuoting::None, false), + (Dialect::Unknown, BarQuoting::Anywhere, false), + ]; + assert_eq!( + expected.len(), + Dialect::ALL.len(), + "a dialect was added without a decision here" + ); + + for (dialect, bar, one_char) in expected { + let policy = DialectReaderPolicy::new(dialect); + assert_eq!(policy.bar_quoting(), bar, "{}", dialect.label()); + assert_eq!( + policy.character_literal_is_exactly_one_char(), + one_char, + "{}", + dialect.label() + ); + } +} diff --git a/packages/core/syntax/src/sexpr/tree.rs b/packages/core/syntax/src/sexpr/tree.rs index 9645ac2f..4a5b4436 100644 --- a/packages/core/syntax/src/sexpr/tree.rs +++ b/packages/core/syntax/src/sexpr/tree.rs @@ -198,12 +198,35 @@ pub enum ReaderPrefix { UnquoteSplicing, Function, ReadEval, - /// A bare `#` immediately before an open delimiter: Common Lisp/Scheme - /// vector literals (`#(1 2 3)`) and Clojure set (`#{1 2}`) or anonymous - /// function (`#(+ % 1)`) literals. All three dialects glue `#` directly - /// onto the following collection with no space, so this keeps the `#` - /// attached to its list instead of scanning as a disconnected atom. + /// A bare `#` immediately before the literal it introduces: Common + /// Lisp/Scheme vector literals (`#(1 2 3)`), Clojure set (`#{1 2}`) or + /// anonymous function (`#(+ % 1)`) literals, LFE tuples (`#(a b)`) and LFE + /// binary strings (`#"GET"`). Every one of them glues `#` directly onto the + /// following literal with no space, so this keeps the `#` attached instead + /// of scanning as a disconnected atom. HashLiteral, + /// LFE's binary literal opener, `#B(…)` or `#b(…)`. + /// + /// `lfe_scan.erl`'s `scan_hash2` returns `'#B('` as a *single* opening + /// token, and `lfe_parse.spell1` closes it with an ordinary `)` + /// (`sexpr -> '#B(' proper_list ')'`). Structurally that is the same shape + /// as `#(`, which is why it is a prefix on the list rather than a token of + /// its own — but it needs its own spelling, because rendering it as a bare + /// `#` would turn a binary into a tuple. + LfeBinary, + /// LFE's map literal opener, `#M(…)` or `#m(…)`. + /// + /// `sexpr -> '#M(' proper_list ')'` in `lfe_parse.spell1`; the elements are + /// alternating keys and values. + LfeMap, + /// LFE's struct literal opener, `#S(…)` or `#s(…)`. + /// + /// `lfe_scan.erl` scans `'#S('` and `lfe_parse.spell1` declares it a + /// terminal, but 2.2.0 has no production that uses it, so LFE's own parser + /// answers `{illegal,'#S('}`. It is lexed here anyway: the alternative is + /// to keep orphaning the `#S` from its list, which is worse for a file that + /// is already broken, and the spelling is fixed by the scanner. + LfeStruct, /// Clojure metadata sugar (`^{:doc "x"}`, `^:private`, `^String`) /// prefixing the map, keyword, or symbol that carries the metadata. Metadata, @@ -225,6 +248,16 @@ impl ReaderPrefix { Self::Function => "#'", Self::ReadEval => "#.", Self::HashLiteral => "#", + // Upper case because that is what LFE's own printer emits + // (`lfe_io_write.erl` writes `["#B(",bytes(Bit, D),$)]`). The + // reader accepts either case, so a document written `#b(` keeps + // its own spelling wherever the source span is used — which is + // every structural edit and the formatter. Only the small number + // of call sites that *synthesise* prefix text from this constant + // normalise the case, and `#b(` and `#B(` are the same literal. + Self::LfeBinary => "#B", + Self::LfeMap => "#M", + Self::LfeStruct => "#S", Self::Metadata => "^", Self::ReaderConditional => "#?", Self::ReaderConditionalSplicing => "#?@", diff --git a/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs b/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs index 20dc1107..9e71d3f5 100644 --- a/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs +++ b/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs @@ -255,7 +255,16 @@ fn render_prefixed_expression( )? )), ReaderPrefix::ReadEval => Ok(view.span.slice(input).to_owned()), + // LFE's `#B(`/`#M(`/`#S(` render the same way: the dispatch, then the + // list. `as_source` spells them upper case, so a body written `#b(…)` + // comes back `#B(…)`; LFE's scanner accepts either and its own printer + // writes upper case, so this normalises the spelling of a literal + // rather than changing it. Every span-based path -- the formatter and + // all structural edits -- keeps the original bytes. ReaderPrefix::HashLiteral + | ReaderPrefix::LfeBinary + | ReaderPrefix::LfeMap + | ReaderPrefix::LfeStruct | ReaderPrefix::Metadata | ReaderPrefix::ReaderConditional | ReaderPrefix::ReaderConditionalSplicing => Ok(format!(