diff --git a/packages/core/syntax/src/sexpr/parser.rs b/packages/core/syntax/src/sexpr/parser.rs index 0b95babe..7337eb41 100644 --- a/packages/core/syntax/src/sexpr/parser.rs +++ b/packages/core/syntax/src/sexpr/parser.rs @@ -2,7 +2,7 @@ use thiserror::Error; use crate::dialect::Dialect; -use super::reader_policy::{DialectReaderPolicy, LongStringExtent, ReaderMacro}; +use super::reader_policy::{BarQuoting, DialectReaderPolicy, LongStringExtent, ReaderMacro}; use super::tree::{Comment, Node, NodeKind, ReaderPrefix, ReaderPrefixes, SyntaxTree}; use super::types::{ByteOffset, ByteSpan, Delimiter, NodeId}; @@ -448,16 +448,31 @@ impl<'a> Parser<'a> { } fn consume_atom_body(&mut self) -> std::result::Result<(), ParseError> { - self.consume_character_literal(); + let token_start = self.pos.get(); + if self.consume_character_literal() { + return Ok(()); + } while self.pos.get() < self.bytes.len() { let byte = self.current_byte(); if self.policy.supports_single_escape() && byte == b'\\' { self.consume_single_escape()?; continue; } - if self.policy.supports_bar_quoted_symbols() && byte == b'|' { - self.consume_multiple_escape()?; - continue; + if byte == b'|' { + match self.policy.bar_quoting() { + BarQuoting::Anywhere => { + self.consume_multiple_escape()?; + continue; + } + // LFE opens a quoted symbol only at a token's first byte; + // anywhere else the `|` is an ordinary constituent, so + // `a|b c|d` stays two symbols rather than becoming one. + BarQuoting::TokenStart if self.pos.get() == token_start => { + self.consume_multiple_escape()?; + continue; + } + BarQuoting::TokenStart | BarQuoting::None => {} + } } if self.policy.is_atom_boundary(self.bytes, self.pos.get()) { break; @@ -467,18 +482,28 @@ impl<'a> Parser<'a> { Ok(()) } - fn consume_character_literal(&mut self) { + /// Consumes a character literal at the cursor, if one starts there. + /// + /// Returns whether the literal is *complete* — whether the token ends where + /// the literal does, so no further scanning may happen. Only LFE says yes: + /// its literal is the two-byte `#\` and exactly one character, with no + /// named forms and no escapes, so `#\"` ends at the quote and the string + /// that follows is a separate token. Elsewhere the name may be several + /// characters (`#\space`, `?\C-x`, `\newline`) and the caller must keep + /// scanning to the next boundary. + fn consume_character_literal(&mut self) -> bool { let Some(prefix_width) = self .policy .character_literal_prefix_width(self.bytes, self.pos.get()) else { - return; + return false; }; self.advance_by(prefix_width); let Some(character) = self.input[self.pos.get()..].chars().next() else { - return; + return false; }; self.advance_by(character.len_utf8()); + self.policy.character_literal_is_exactly_one_char() } /// Consumes a Lisp single-escape (`\`) and the following character literally. diff --git a/packages/core/syntax/src/sexpr/reader.rs b/packages/core/syntax/src/sexpr/reader.rs index bee8a8b3..dd6046b4 100644 --- a/packages/core/syntax/src/sexpr/reader.rs +++ b/packages/core/syntax/src/sexpr/reader.rs @@ -46,7 +46,15 @@ pub fn apply_reader_prefix_context( // anonymous functions and metadata targets), so treat them like // `Function` rather than opaque data: keep traversing normally // instead of hiding the contents from rename/reference tracking. + // LFE's `#B(…)`, `#M(…)` and `#S(…)` sit here for the same reason: + // their elements are ordinary expressions, not opaque data. A + // binary segment is `(value (size N) (unit 8))` and a map literal's + // values are arbitrary forms, so rename and reference tracking must + // keep descending into them. ReaderPrefix::HashLiteral + | ReaderPrefix::LfeBinary + | ReaderPrefix::LfeMap + | ReaderPrefix::LfeStruct | ReaderPrefix::Metadata | ReaderPrefix::ReaderConditional | ReaderPrefix::ReaderConditionalSplicing => {} diff --git a/packages/core/syntax/src/sexpr/reader_policy.rs b/packages/core/syntax/src/sexpr/reader_policy.rs index 5bc62004..f1a5abe4 100644 --- a/packages/core/syntax/src/sexpr/reader_policy.rs +++ b/packages/core/syntax/src/sexpr/reader_policy.rs @@ -21,6 +21,18 @@ pub(super) enum ReaderMacro { }, } +/// Where a `|...|` region may begin inside a token. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum BarQuoting { + /// A `|` is an ordinary symbol constituent wherever it appears. + None, + /// CLHS 2.1.4.2 / R7RS 2.1: a multiple-escape may open anywhere in a token. + Anywhere, + /// LFE: a `|` opens a quoted symbol only where a token starts; anywhere + /// else it is an ordinary constituent. + TokenStart, +} + /// How far a Janet long string reaches from its opening backtick run. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum LongStringExtent { @@ -105,15 +117,31 @@ impl DialectReaderPolicy { ) } - /// Whether `|...|` reads as one symbol rather than a token boundary. + /// Where `|...|` reads as one symbol rather than a token boundary. /// /// R7RS 2.1 gives Scheme the same vertical-line notation Common Lisp has - /// in CLHS 2.1.4.2, so `|Foo Bar|` is a single identifier in both. - pub(super) const fn supports_bar_quoted_symbols(self) -> bool { - matches!( - self.dialect, - Dialect::CommonLisp | Dialect::Scheme | Dialect::Racket | Dialect::Unknown - ) + /// in CLHS 2.1.4.2, so `|Foo Bar|` is a single identifier in both, and in + /// both a `|` may open a quoted region part-way through a token. + /// + /// LFE has the notation but not that second property, and the difference + /// is explicit in `lfe_scan.erl`: `start_symbol_char($|) -> false` sends a + /// leading `|` to `scan_qsymbol`, while `symbol_char/1` has no `$|` clause + /// at all, so it falls through to `(C > $\s) and (C =< $~)` — true for + /// `|` (124). A `|` inside a token is therefore an ordinary constituent + /// and `a|b c|d` is the two symbols `a|b` and `c|d`, not one. + pub(super) const fn bar_quoting(self) -> BarQuoting { + match self.dialect { + Dialect::CommonLisp | Dialect::Scheme | Dialect::Racket | Dialect::Unknown => { + BarQuoting::Anywhere + } + Dialect::Lfe => BarQuoting::TokenStart, + Dialect::EmacsLisp + | Dialect::Clojure + | Dialect::Hy + | Dialect::Carp + | Dialect::Janet + | Dialect::Fennel => BarQuoting::None, + } } /// Whether a bare `\` escapes the next character *outside* `|...|`. @@ -231,7 +259,11 @@ impl DialectReaderPolicy { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); let width = match self.dialect { - Dialect::Scheme | Dialect::Racket if byte == b'#' && next == Some(b'\\') => 2, + Dialect::Scheme | Dialect::Racket | Dialect::Lfe + if byte == b'#' && next == Some(b'\\') => + { + 2 + } Dialect::Clojure if byte == b'\\' => 1, Dialect::EmacsLisp if byte == b'?' && next == Some(b'\\') => 2, Dialect::EmacsLisp if byte == b'?' => 1, @@ -240,15 +272,40 @@ impl DialectReaderPolicy { bytes.get(pos + width).is_some().then_some(width) } + /// Whether a character literal is *exactly* its prefix plus one character, + /// so the token ends there rather than running on to the next boundary. + /// + /// LFE is the only dialect here where it is. `lfe_scan.erl` spells the + /// whole grammar in one clause: + /// + /// ```erlang + /// scan_hash2([$\\,C|Cs], Line, Col, [], St) -> + /// {ok,{number,Line,C},Cs,Line,Col+2,St}; + /// ``` + /// + /// One character, taken verbatim. There are no named characters, and no + /// escape processing at all — `#\n` is the letter `n` (110), not a newline. + /// So `#\"abc"` is the character `"` followed by the string `abc`, and + /// running the token on to the next boundary instead would swallow the + /// string's opening quote and glue the rest of the file into one atom. + /// + /// Everywhere else the name may be longer than one character — Scheme's + /// `#\space`, Emacs Lisp's `?\C-x`, Clojure's `\newline` — so the token has + /// to keep scanning, and nothing below may change for them. + pub(super) const fn character_literal_is_exactly_one_char(self) -> bool { + matches!(self.dialect, Dialect::Lfe) + } + pub(super) fn classify_reader_macro(self, bytes: &[u8], pos: usize) -> Option { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); let third = bytes.get(pos + 2).copied(); match self.dialect { - Dialect::Unknown | Dialect::Lfe | Dialect::Hy | Dialect::Carp => { + Dialect::Unknown | Dialect::Hy | Dialect::Carp => { self.classify_legacy(byte, next, third) } + Dialect::Lfe => self.classify_lfe(bytes, pos), Dialect::CommonLisp => self.classify_common_lisp(bytes, pos), Dialect::EmacsLisp => self.classify_emacs_lisp(bytes, pos), Dialect::Scheme | Dialect::Racket => self.classify_scheme(bytes, pos), @@ -300,6 +357,85 @@ impl DialectReaderPolicy { classify_shared_prefix(byte, next, third) } + /// LFE's `#`-dispatch table, from `lfe_scan.erl`'s `scan_hash1`/`scan_hash2`. + /// + /// That is the complete set, in the scanner's own order. `scan_hash` + /// collects decimal digits first (`scan_hash_digits`), and every form + /// except `#r` requires that digit run to be empty: + /// + /// | source | token | handled by | + /// |---------------|---------------------------|-------------------------------| + /// | `#(` | `'#('` tuple open | [`classify_shared_prefix`] | + /// | `#B(` `#b(` | `'#B('` binary open | this arm | + /// | `#M(` `#m(` | `'#M('` map open | this arm | + /// | `#S(` `#s(` | `'#S('` struct open | this arm | + /// | `#"…"` | `binary` string | this arm | + /// | `#\C` | `{number,_,C}` one char | `character_literal_prefix_width` | + /// | `#'f/2` | `'#\''` fun reference | [`classify_shared_prefix`] | + /// | `#.` | `'#.'` read-eval | [`classify_shared_prefix`] | + /// | `#\|` | block comment | `supports_block_comments` | + /// | `#*1010` | base-2 number | scans as a plain atom | + /// | `#b…` `#o…` `#d…` `#x…` | based numbers | scan as plain atoms | + /// | `#r…` | base-2..36 number | scans as a plain atom | + /// | `` #` `` `#;` `#,` `#,@` | scanned, no grammar production | left as they were | + /// + /// The based-number forms need nothing: `#x1f` has no delimiter in it, so + /// the ordinary atom scanner already takes the whole token. The last row is + /// deliberately untouched — `lfe_parse.spell1` declares no production for + /// those tokens, so LFE itself refuses them, and the existing readings are + /// neither more nor less wrong than they were. + /// + /// Everything else falls through to [`Self::classify_legacy`], which is + /// what LFE used before this function existed, so no reading changes except + /// the ones named above. + fn classify_lfe(self, bytes: &[u8], pos: usize) -> Option { + let byte = *bytes.get(pos)?; + let next = bytes.get(pos + 1).copied(); + let third = bytes.get(pos + 2).copied(); + + if byte != b'#' { + return self.classify_legacy(byte, next, third); + } + + // `#B(`, `#M(`, `#S(` are single opening tokens in `scan_hash2`, each + // closed by a plain `)` in `lfe_parse.spell1`. Reading the two-byte + // dispatch as a prefix on the list that follows gives exactly that + // shape, and is how `#(` has always been read. Without it the `#B` + // scanned as its own atom and the list became a *sibling*, so + // `(f #B(1 2) X)` had four children where LFE sees three -- silently, + // at exit 0, which is why this is the defect that matters most. + if third == Some(b'(') { + match next { + Some(b'b' | b'B') => return prefix(ReaderPrefix::LfeBinary, 2), + Some(b'm' | b'M') => return prefix(ReaderPrefix::LfeMap, 2), + Some(b's' | b'S') => return prefix(ReaderPrefix::LfeStruct, 2), + _ => {} + } + } + + // `#"…"` is one `binary` token (`scan_hash1([$"|Cs], …)` hands + // straight to `scan_binary_string`). Treating the `#` as a prefix on + // the string that follows keeps the whole literal in one node, where + // before the `#` glued onto the string's *first word* and every later + // word became a sibling atom: `#"text/plain; version=0.0.4"` split + // into three, and one containing a `)` closed its enclosing list early. + if next == Some(b'"') { + return prefix(ReaderPrefix::HashLiteral, 1); + } + + // `#\` with nothing after it is a truncated character literal. + // Refusing it is what stops the formatter's trailing newline from + // becoming the literal's character on the next parse, which would make + // `format(format(x))` differ from `format(x)`. Scheme and Racket + // already refuse the same input for the same reason; LFE's own scanner + // refuses it too, as `{illegal_token,"#\\"}`. + if next == Some(b'\\') && third.is_none() { + return Some(ReaderMacro::UnsupportedDispatch { width: 1 }); + } + + self.classify_legacy(byte, next, third) + } + fn classify_common_lisp(self, bytes: &[u8], pos: usize) -> Option { let byte = *bytes.get(pos)?; let next = bytes.get(pos + 1).copied(); diff --git a/packages/core/syntax/src/sexpr/tests/parser.rs b/packages/core/syntax/src/sexpr/tests/parser.rs index fa001fc9..22ddb4c0 100644 --- a/packages/core/syntax/src/sexpr/tests/parser.rs +++ b/packages/core/syntax/src/sexpr/tests/parser.rs @@ -1681,3 +1681,339 @@ fn clojure_namespaced_map_width_stops_at_the_namespace() { // Comments are not whitespace to Clojure's reader here. assert_eq!(width("#:foo ;; c\n{:a 1}"), None); } + +// --------------------------------------------------------------------------- +// LFE reader +// +// The reference is LFE 2.2.0's own scanner, `src/lfe_scan.erl`, and its +// grammar, `src/lfe_parse.spell1`. Every expected reading below is that +// scanner's token stream, checked by running the case through +// `lfe_scan:string/1` -- not a guess at what the notation ought to mean. +// --------------------------------------------------------------------------- + +/// The source text of every child of the document's single top-level list. +fn lfe_children(input: &'static str) -> Vec<&'static str> { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect() +} + +/// `#B(`, `#M(` and `#S(` are single *opening* tokens in `scan_hash2`: +/// +/// ```erlang +/// scan_hash2([C,$\(|Cs], Line, Col, [], St) when (C =:= $b) or (C =:= $B) -> +/// {ok,{'#B(',Line},Cs,Line,Col,St}; +/// ``` +/// +/// closed by a plain `)`, since `lfe_parse.spell1` reads +/// `sexpr -> '#B(' proper_list ')'`. Taking the two-byte dispatch as a prefix +/// on the following list gives exactly that shape. +/// +/// Before this, the `#B` scanned as its own atom and the list became its +/// *sibling*, so `(f #B(1 2) X)` had four children where LFE sees three -- +/// silently, at exit 0, which made every arity-sensitive rule read the form +/// wrong. +#[test] +fn lfe_hash_letter_collections_stay_attached_to_their_list() { + struct Case { + input: &'static str, + children: &'static [&'static str], + prefix: ReaderPrefix, + } + + let cases = [ + Case { + input: "(f #B(1 2) X)", + children: &["f", "#B(1 2)", "X"], + prefix: ReaderPrefix::LfeBinary, + }, + // The scanner accepts either case, and the document keeps its own. + Case { + input: "(f #b(1 2) X)", + children: &["f", "#b(1 2)", "X"], + prefix: ReaderPrefix::LfeBinary, + }, + Case { + input: "(g #M(a 1 b 2) Y)", + children: &["g", "#M(a 1 b 2)", "Y"], + prefix: ReaderPrefix::LfeMap, + }, + Case { + input: "(g #m(a 1) Y)", + children: &["g", "#m(a 1)", "Y"], + prefix: ReaderPrefix::LfeMap, + }, + // `lfe_scan` emits `'#S('` and `lfe_parse.spell1` declares it a + // terminal, but 2.2.0 has no production using it, so LFE itself + // answers `{illegal,'#S('}`. Lexing it anyway beats orphaning the `#S` + // from its list in a file that is already broken. + Case { + input: "(h #S(point x 1) Z)", + children: &["h", "#S(point x 1)", "Z"], + prefix: ReaderPrefix::LfeStruct, + }, + // The tuple opener, which has always worked, pinned so the new arms + // cannot displace it. + Case { + input: "(i #(1 2) W)", + children: &["i", "#(1 2)", "W"], + prefix: ReaderPrefix::HashLiteral, + }, + ]; + + for case in cases { + assert_eq!(lfe_children(case.input), case.children, "{}", case.input); + let tree = SyntaxTree::parse_with_dialect(case.input, Dialect::Lfe).expect("valid"); + let literal = &tree.root_view().children[0].children[1]; + assert_eq!(literal.kind, ExpressionKind::List, "{}", case.input); + assert_eq!(literal.reader_prefixes, vec![case.prefix], "{}", case.input); + } +} + +/// `#b` and its friends only open a collection when a `(` follows. +/// `scan_hash2` orders the binary-token clause before the based-number one for +/// exactly this reason ("Scan binary tokens, these must come before the based +/// number"), so `#b1010` stays the number ten. +#[test] +fn lfe_based_numbers_are_not_collection_openers() { + for input in [ + "(f #b1010 x)", + "(f #B1010 x)", + "(f #x1f x)", + "(f #o17 x)", + "(f #d99 x)", + "(f #2r1010 x)", + "(f #*1010 x)", + ] { + let children = lfe_children(input); + assert_eq!(children.len(), 3, "{input}"); + assert_eq!(children[0], "f", "{input}"); + assert_eq!(children[2], "x", "{input}"); + } +} + +/// `scan_hash1([$"|Cs], Line, Col, [], St) -> scan_binary_string(...)` makes +/// `#"..."` a single `binary` token. +/// +/// Before this the `#` glued onto the string's *first word* and every later +/// word became a sibling atom, so `#"text/plain; version=0.0.4"` split into +/// three and one containing a `)` closed its enclosing list early. Binary +/// strings are the most common of these constructs in real LFE. +#[test] +fn lfe_binary_strings_are_one_atom() { + let cases: &[(&str, &[&str])] = &[ + ("(f #\"GET\" x)", &["f", "#\"GET\"", "x"]), + ("(f #\"a b\" x)", &["f", "#\"a b\"", "x"]), + ("(f #\"x)y\" x)", &["f", "#\"x)y\"", "x"]), + ("(f #\"\" x)", &["f", "#\"\"", "x"]), + ( + "(f #\"text/plain; version=0.0.4\" x)", + &["f", "#\"text/plain; version=0.0.4\"", "x"], + ), + ("(f #\"esc \\\" q\" x)", &["f", "#\"esc \\\" q\"", "x"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// LFE's whole character-literal grammar is one clause: +/// +/// ```erlang +/// scan_hash2([$\\,C|Cs], Line, Col, [], St) -> +/// {ok,{number,Line,C},Cs,Line,Col+2,St}; +/// ``` +/// +/// Two bytes of prefix and exactly one character, taken verbatim. There are no +/// named characters and no escape processing at all, so `#\newline` is the +/// letter `n` followed by the symbol `ewline` -- `lfe_scan:string/1` answers +/// `[{number,1,110},{symbol,1,ewline}]` for it. +#[test] +fn lfe_character_literal_is_exactly_one_character() { + let cases: &[(&str, &[&str])] = &[ + // The delimiters. These used to restructure the tree outright: `#\(` + // scanned as the atom `#\` and then *opened a list*. + ("(list #\\( #\\))", &["list", "#\\(", "#\\)"]), + // A `;` would otherwise start a comment and eat the rest of the line. + ("(list #\\; a)", &["list", "#\\;", "a"]), + // A `"` would otherwise open a string and swallow the file. + ("(list #\\\" a)", &["list", "#\\\"", "a"]), + ("(list #\\\\ a)", &["list", "#\\\\", "a"]), + ("(list #\\| a)", &["list", "#\\|", "a"]), + ("(list #\\a #\\b)", &["list", "#\\a", "#\\b"]), + // One character means one character: the rest is a separate symbol. + ("(f #\\newline)", &["f", "#\\n", "ewline"]), + // A multi-byte character is one character, not one byte. + ("(f #\\\u{e9} x)", &["f", "#\\\u{e9}", "x"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// `start_symbol_char($|) -> false` sends a *leading* `|` to `scan_qsymbol`, +/// which runs to the closing `|` taking `\C` verbatim; whitespace and +/// delimiters inside are ordinary content. But `symbol_char/1` has no `$|` +/// clause, so it falls through to `(C > $\s) and (C =< $~)` -- true for `|` +/// (124) -- and a `|` *inside* a token is an ordinary constituent. +/// +/// Both halves matter. Without the first, `'|foo bar|` split at the space; +/// with the first but not the second, `a|b c|d` would fuse into one symbol. +#[test] +fn lfe_bar_quoted_symbols_open_only_at_a_token_start() { + let cases: &[(&str, &[&str])] = &[ + ("(f '|foo bar|)", &["f", "'|foo bar|"]), + ("(f |a(b| x)", &["f", "|a(b|", "x"]), + ("(f |x\\|y| x)", &["f", "|x\\|y|", "x"]), + ("(f || x)", &["f", "||", "x"]), + ("(f |;| x)", &["f", "|;|", "x"]), + // A newline inside is content: `scan_qsymbol1` has an explicit `$\n` + // clause that keeps collecting. + ("(f |multi\nline| x)", &["f", "|multi\nline|", "x"]), + // Mid-token, an ordinary constituent. Two symbols, not one. + ("(f a|b c|d)", &["f", "a|b", "c|d"]), + ]; + + for (input, expected) in cases { + let tree = SyntaxTree::parse_with_dialect(input, Dialect::Lfe) + .unwrap_or_else(|error| panic!("{input:?}: {error}")); + let children: Vec<&str> = tree.root_view().children[0] + .children + .iter() + .map(|child| child.span.slice(input)) + .collect(); + assert_eq!(&children, expected, "{input}"); + } +} + +/// An unterminated `|` fails loudly rather than consuming the rest of the file +/// as one symbol. +/// +/// LFE refuses it too -- `scan_qsymbol1(eof, ...)` raises +/// `{illegal_chars,[$| | Symcs]}` -- so this agrees with the reference reader +/// rather than inventing a rule. Reading to EOF as one atom would be exactly +/// the silent corruption this work exists to remove. +#[test] +fn lfe_unterminated_bar_quoted_symbol_is_refused() { + let error = SyntaxTree::parse_with_dialect("(f |unterminated\n(g 1)\n", Dialect::Lfe) + .expect_err("unterminated multiple escape"); + assert_eq!(error, ParseError::UnterminatedSymbol(3)); +} + +/// `#\` with nothing after it is a truncated literal, not the character for +/// nothing. LFE answers `{illegal_token,"#\\"}`. +/// +/// Accepting it as a complete atom would make the formatter non-idempotent: +/// it appends a trailing newline, the truncated literal claims it as its +/// character, and the next pass appends another. Scheme and Racket already +/// refuse the same input for the same reason. +#[test] +fn lfe_truncated_character_literal_is_refused() { + let error = SyntaxTree::parse_with_dialect("(f #\\", Dialect::Lfe) + .expect_err("truncated character literal"); + assert_eq!( + error, + ParseError::UnsupportedReaderDispatch { + dispatch: "#".to_owned(), + position: 3, + } + ); +} + +/// Everything above is LFE-only. +/// +/// `#(` is a vector in Scheme and Racket and a set or lambda in Clojure, so +/// these bytes are live elsewhere with other meanings; a letter between the +/// `#` and the `(` means something different or nothing in all ten. This pins +/// them against a later edit widening `classify_lfe`'s arms by accident. +#[test] +fn lfe_hash_letter_collections_do_not_leak_into_other_dialects() { + for dialect in [ + Dialect::CommonLisp, + Dialect::EmacsLisp, + Dialect::Scheme, + Dialect::Racket, + Dialect::Clojure, + Dialect::Fennel, + Dialect::Janet, + Dialect::Hy, + Dialect::Carp, + Dialect::Unknown, + ] { + let label = dialect.label(); + // Refusing the input is fine; what must not happen is reading it as + // one prefixed list the way LFE now does. + let Ok(tree) = SyntaxTree::parse_with_dialect("(f #B(1 2) X)", dialect) else { + continue; + }; + let prefixes: Vec = tree.root_view().children[0] + .children + .iter() + .flat_map(|child| child.reader_prefixes.clone()) + .collect(); + assert!( + !prefixes.contains(&ReaderPrefix::LfeBinary), + "{label} produced an LFE binary literal" + ); + } +} + +/// The two lexical switches this change added, pinned per dialect. +/// +/// The table is worth more than the prose: `bar_quoting` replaced a boolean, +/// and its mapping has to stay exactly what that boolean was for the ten +/// dialects LFE is not. +#[test] +fn lfe_lexical_switches_are_scoped_to_lfe() { + use crate::sexpr::reader_policy::BarQuoting; + + let expected = [ + (Dialect::CommonLisp, BarQuoting::Anywhere, false), + (Dialect::EmacsLisp, BarQuoting::None, false), + (Dialect::Lfe, BarQuoting::TokenStart, true), + (Dialect::Scheme, BarQuoting::Anywhere, false), + (Dialect::Racket, BarQuoting::Anywhere, false), + (Dialect::Clojure, BarQuoting::None, false), + (Dialect::Hy, BarQuoting::None, false), + (Dialect::Carp, BarQuoting::None, false), + (Dialect::Janet, BarQuoting::None, false), + (Dialect::Fennel, BarQuoting::None, false), + (Dialect::Unknown, BarQuoting::Anywhere, false), + ]; + assert_eq!( + expected.len(), + Dialect::ALL.len(), + "a dialect was added without a decision here" + ); + + for (dialect, bar, one_char) in expected { + let policy = DialectReaderPolicy::new(dialect); + assert_eq!(policy.bar_quoting(), bar, "{}", dialect.label()); + assert_eq!( + policy.character_literal_is_exactly_one_char(), + one_char, + "{}", + dialect.label() + ); + } +} diff --git a/packages/core/syntax/src/sexpr/tree.rs b/packages/core/syntax/src/sexpr/tree.rs index 9645ac2f..4a5b4436 100644 --- a/packages/core/syntax/src/sexpr/tree.rs +++ b/packages/core/syntax/src/sexpr/tree.rs @@ -198,12 +198,35 @@ pub enum ReaderPrefix { UnquoteSplicing, Function, ReadEval, - /// A bare `#` immediately before an open delimiter: Common Lisp/Scheme - /// vector literals (`#(1 2 3)`) and Clojure set (`#{1 2}`) or anonymous - /// function (`#(+ % 1)`) literals. All three dialects glue `#` directly - /// onto the following collection with no space, so this keeps the `#` - /// attached to its list instead of scanning as a disconnected atom. + /// A bare `#` immediately before the literal it introduces: Common + /// Lisp/Scheme vector literals (`#(1 2 3)`), Clojure set (`#{1 2}`) or + /// anonymous function (`#(+ % 1)`) literals, LFE tuples (`#(a b)`) and LFE + /// binary strings (`#"GET"`). Every one of them glues `#` directly onto the + /// following literal with no space, so this keeps the `#` attached instead + /// of scanning as a disconnected atom. HashLiteral, + /// LFE's binary literal opener, `#B(…)` or `#b(…)`. + /// + /// `lfe_scan.erl`'s `scan_hash2` returns `'#B('` as a *single* opening + /// token, and `lfe_parse.spell1` closes it with an ordinary `)` + /// (`sexpr -> '#B(' proper_list ')'`). Structurally that is the same shape + /// as `#(`, which is why it is a prefix on the list rather than a token of + /// its own — but it needs its own spelling, because rendering it as a bare + /// `#` would turn a binary into a tuple. + LfeBinary, + /// LFE's map literal opener, `#M(…)` or `#m(…)`. + /// + /// `sexpr -> '#M(' proper_list ')'` in `lfe_parse.spell1`; the elements are + /// alternating keys and values. + LfeMap, + /// LFE's struct literal opener, `#S(…)` or `#s(…)`. + /// + /// `lfe_scan.erl` scans `'#S('` and `lfe_parse.spell1` declares it a + /// terminal, but 2.2.0 has no production that uses it, so LFE's own parser + /// answers `{illegal,'#S('}`. It is lexed here anyway: the alternative is + /// to keep orphaning the `#S` from its list, which is worse for a file that + /// is already broken, and the spelling is fixed by the scanner. + LfeStruct, /// Clojure metadata sugar (`^{:doc "x"}`, `^:private`, `^String`) /// prefixing the map, keyword, or symbol that carries the metadata. Metadata, @@ -225,6 +248,16 @@ impl ReaderPrefix { Self::Function => "#'", Self::ReadEval => "#.", Self::HashLiteral => "#", + // Upper case because that is what LFE's own printer emits + // (`lfe_io_write.erl` writes `["#B(",bytes(Bit, D),$)]`). The + // reader accepts either case, so a document written `#b(` keeps + // its own spelling wherever the source span is used — which is + // every structural edit and the formatter. Only the small number + // of call sites that *synthesise* prefix text from this constant + // normalise the case, and `#b(` and `#B(` are the same literal. + Self::LfeBinary => "#B", + Self::LfeMap => "#M", + Self::LfeStruct => "#S", Self::Metadata => "^", Self::ReaderConditional => "#?", Self::ReaderConditionalSplicing => "#?@", diff --git a/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs b/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs index 20dc1107..9e71d3f5 100644 --- a/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs +++ b/packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs @@ -255,7 +255,16 @@ fn render_prefixed_expression( )? )), ReaderPrefix::ReadEval => Ok(view.span.slice(input).to_owned()), + // LFE's `#B(`/`#M(`/`#S(` render the same way: the dispatch, then the + // list. `as_source` spells them upper case, so a body written `#b(…)` + // comes back `#B(…)`; LFE's scanner accepts either and its own printer + // writes upper case, so this normalises the spelling of a literal + // rather than changing it. Every span-based path -- the formatter and + // all structural edits -- keeps the original bytes. ReaderPrefix::HashLiteral + | ReaderPrefix::LfeBinary + | ReaderPrefix::LfeMap + | ReaderPrefix::LfeStruct | ReaderPrefix::Metadata | ReaderPrefix::ReaderConditional | ReaderPrefix::ReaderConditionalSplicing => Ok(format!(