diff --git a/packages/core/syntax/src/sexpr/parser.rs b/packages/core/syntax/src/sexpr/parser.rs index dc0d4c53..0b95babe 100644 --- a/packages/core/syntax/src/sexpr/parser.rs +++ b/packages/core/syntax/src/sexpr/parser.rs @@ -2,7 +2,7 @@ use thiserror::Error; use crate::dialect::Dialect; -use super::reader_policy::{DialectReaderPolicy, ReaderMacro}; +use super::reader_policy::{DialectReaderPolicy, LongStringExtent, ReaderMacro}; use super::tree::{Comment, Node, NodeKind, ReaderPrefix, ReaderPrefixes, SyntaxTree}; use super::types::{ByteOffset, ByteSpan, Delimiter, NodeId}; @@ -408,6 +408,35 @@ impl<'a> Parser<'a> { Err(ParseError::UnterminatedString(start.get())) } + fn atom_long_string_with_prefixes( + &mut self, + prefixes: Vec, + ) -> std::result::Result<(), ParseError> { + let start = self.pos; + let width = self.long_string_width(start.get())?; + self.advance_by(width); + self.push_atom(prefixes, start, self.pos); + Ok(()) + } + + /// Byte width of the Janet long string opening at `start`. + /// + /// An unterminated one is refused rather than read to EOF. Janet refuses + /// it too: `janet_parser_eof` finds the `longstring` state still on the + /// stack and reports "unexpected end of source". Reading it as an atom + /// instead would hand every later command a tree in which the remainder + /// of the file is one giant symbol -- silent corruption of exactly the + /// kind this fix exists to remove -- so it fails loudly, and reuses the + /// error the `"..."` path already raises for the same shape of mistake. + fn long_string_width(&self, start: usize) -> std::result::Result { + match self.policy.long_string_extent(self.bytes, start) { + Some(LongStringExtent::Closed { width }) => Ok(width), + Some(LongStringExtent::Unterminated) | None => { + Err(ParseError::UnterminatedString(start)) + } + } + } + fn atom_with_prefixes( &mut self, prefixes: Vec, @@ -629,6 +658,9 @@ impl<'a> Parser<'a> { return Err(self.raw_delimiter_error()); } b'"' => self.atom_string_with_prefixes(prefixes)?, + b'`' if self.policy.has_long_strings() => { + self.atom_long_string_with_prefixes(prefixes)?; + } _ => self.atom_with_prefixes(prefixes)?, } Ok(()) @@ -806,6 +838,19 @@ impl<'a> Parser<'a> { return Err(self.raw_delimiter_error()); } b'"' => self.skip_string()?, + // Unreachable under Janet today: `#` is always a line + // comment there, so `classify_janet` never returns a + // `Discard` and nothing enters this scanner. It is + // here because `DialectReaderPolicy` exists to stop + // the recording and scanning paths disagreeing about + // how far a reader form reaches, and a Janet datum + // comment added later must not silently reintroduce + // that disagreement. Both paths call the same + // `long_string_extent`, which is what makes them agree. + b'`' if self.policy.has_long_strings() => { + let width = self.long_string_width(self.pos.get())?; + self.advance_by(width); + } _ => self.skip_atom()?, } } diff --git a/packages/core/syntax/src/sexpr/reader_policy.rs b/packages/core/syntax/src/sexpr/reader_policy.rs index f68e36aa..3abe869c 100644 --- a/packages/core/syntax/src/sexpr/reader_policy.rs +++ b/packages/core/syntax/src/sexpr/reader_policy.rs @@ -21,6 +21,15 @@ pub(super) enum ReaderMacro { }, } +/// How far a Janet long string reaches from its opening backtick run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum LongStringExtent { + /// Total byte width of the literal, both delimiter runs included. + Closed { width: usize }, + /// An opening run with no closing run of the same length before EOF. + Unterminated, +} + /// Dialect-specific lexical decisions shared by normal parsing and discarded /// form scanning. Keeping these decisions in one place prevents the two paths /// from disagreeing about the extent of a reader form. @@ -133,14 +142,78 @@ impl DialectReaderPolicy { Delimiter::from_open(byte).is_some() || Delimiter::from_close(byte).is_some() } + /// Whether a backtick opens a long string in this dialect. + /// + /// Janet is the only one. Its `root` state sends every backtick to the + /// `longstring` consumer (`src/core/parse.c`), and `symchars` leaves bit + /// 0x60 clear, so a backtick is not a symbol character either -- it both + /// opens a literal and ends whatever token preceded it. + /// + /// In every other dialect here a backtick is quasiquote (Common Lisp, + /// Scheme, Racket, Emacs Lisp, Fennel, and the permissive legacy reader) + /// or syntax-quote (Clojure), which `classify_reader_macro` already + /// returns as a one-byte `ReaderPrefix::Quasiquote`. Nothing below may + /// change for them. + pub(super) const fn has_long_strings(self) -> bool { + matches!(self.dialect, Dialect::Janet) + } + pub(super) fn is_atom_boundary(self, bytes: &[u8], pos: usize) -> bool { bytes.get(pos).is_none_or(|byte| { self.is_whitespace(*byte) || Self::is_raw_delimiter(*byte) + // Dialect first: `self.dialect` is loop-invariant across the + // per-byte calls this makes for every atom in the document, so + // for the nine dialects without long strings the test folds + // away instead of costing a comparison per byte. + || (self.has_long_strings() && *byte == b'`') || self.line_comment_width(bytes, pos).is_some() }) } + /// How far the Janet long string starting at `pos` reaches, if one starts + /// there. + /// + /// Janet's `longstring` state (`src/core/parse.c`) counts the opening run + /// in `argn` while it keeps seeing backticks, then closes the literal on + /// the `argn`-th consecutive backtick it meets afterwards. Two consequences + /// follow, and both are load-bearing: + /// + /// * The opener is the *whole* run. ```` ```` ```` is a four-backtick + /// opener, not two empty strings, so an empty long string cannot be + /// written at all. + /// * The close is exactly `argn` backticks, not at least `argn`. Janet + /// returns 0 from `stringend` so the character that revealed the end is + /// re-dispatched, which means a longer run leaves its surplus to open + /// the next datum: `` ```ab```` x` `` reads as `"ab"` then `" x"`. A run + /// shorter than `argn` is content ("failed end candidate" pushes the + /// backticks it had buffered back into the string). + /// + /// There is no escape processing inside one -- the `PFLAG_INSTRING` branch + /// has no `\\` case -- and a newline is an ordinary content byte, which is + /// the entire point of the form. + pub(super) fn long_string_extent(self, bytes: &[u8], pos: usize) -> Option { + if !self.has_long_strings() || bytes.get(pos) != Some(&b'`') { + return None; + } + let open_len = backtick_run_length(bytes, pos); + let mut cursor = pos + open_len; + while cursor < bytes.len() { + if bytes[cursor] != b'`' { + cursor += 1; + continue; + } + let run = backtick_run_length(bytes, cursor); + if run >= open_len { + return Some(LongStringExtent::Closed { + width: cursor + open_len - pos, + }); + } + cursor += run; + } + Some(LongStringExtent::Unterminated) + } + /// How many bytes introduce a character literal at `pos`, if one starts /// there. /// @@ -482,6 +555,14 @@ impl DialectReaderPolicy { } } +/// How many consecutive backticks start at `pos`. +fn backtick_run_length(bytes: &[u8], pos: usize) -> usize { + bytes[pos..] + .iter() + .take_while(|byte| **byte == b'`') + .count() +} + /// The Racket language directive, which the reader consumes to end of line. pub(crate) const LANG_DIRECTIVE: &str = "#lang"; diff --git a/packages/core/syntax/src/sexpr/tests/parser.rs b/packages/core/syntax/src/sexpr/tests/parser.rs index 5b2f0789..4914454a 100644 --- a/packages/core/syntax/src/sexpr/tests/parser.rs +++ b/packages/core/syntax/src/sexpr/tests/parser.rs @@ -1,6 +1,7 @@ use super::*; use crate::dialect::Dialect; use crate::sexpr::parser::MAX_DISCARDED_FORM_STACK_FRAMES; +use crate::sexpr::reader_policy::{DialectReaderPolicy, LongStringExtent}; #[test] fn parses_balanced_document() { @@ -1225,3 +1226,222 @@ fn find_parse_errors_is_capped_on_pathological_input() { let errors = SyntaxTree::find_parse_errors(&source, Dialect::CommonLisp); assert_eq!(errors.len(), 50, "{errors:?}"); } + +/// Janet's `root` state sends every backtick to the `longstring` consumer +/// (`src/core/parse.c`), so a run of N backticks opens a string that the next +/// run of N backticks closes. Backtick is absent from Janet's `symchars` +/// table too, so it also ends whatever token preceded it. +/// +/// Every expectation below was read off Janet 1.41.3's own reader before it +/// was written here: these are the values `janet -e '(pp (parse ...))'` +/// prints, not a restatement of what this parser happens to do. +#[test] +fn janet_long_strings_are_one_atom() { + struct Case { + input: &'static str, + /// Source text of each child of the single top-level list. + children: &'static [&'static str], + /// What Janet's own reader makes of the literal, for the record. + janet_value: &'static str, + } + + let cases = [ + // A *single* backtick opens one. This is the case most likely to be + // guessed wrong: the rule is not "two or more". + Case { + input: "(f `abc`)", + children: &["f", "`abc`"], + janet_value: r#""abc""#, + }, + Case { + input: "(f ``abc``)", + children: &["f", "``abc``"], + janet_value: r#""abc""#, + }, + // The triple-backtick docstring, the dominant Janet idiom, holding + // the bracket that used to unbalance the whole document. + Case { + input: "(f ```a [b, c) d``` tail)", + children: &["f", "```a [b, c) d```", "tail"], + janet_value: r#""a [b, c) d""#, + }, + // A newline is an ordinary content byte; that is the entire point. + Case { + input: "(f ```line1\nline2```)", + children: &["f", "```line1\nline2```"], + janet_value: r#""line1\nline2""#, + }, + // No escape processing at all: the `PFLAG_INSTRING` branch has no + // `\\` case, so this is a backslash followed by an `n`. + Case { + input: "(f `a\\nb`)", + children: &["f", "`a\\nb`"], + janet_value: r#""a\\nb""#, + }, + // A double quote inside is content, not a nested string. + Case { + input: "(f `say \"hi\"`)", + children: &["f", "`say \"hi\"`"], + janet_value: r#""say \"hi\"""#, + }, + // Runs shorter than the opener are content, not a close. + Case { + input: "(f ```a`b```)", + children: &["f", "```a`b```"], + janet_value: r#""a`b""#, + }, + Case { + input: "(f ```a``b```)", + children: &["f", "```a``b```"], + janet_value: r#""a``b""#, + }, + // The close is exactly N, not at least N. Janet returns 0 from + // `stringend`, so the character that revealed the end is re-dispatched + // and a longer run leaves its surplus to open the *next* datum: Janet + // reads this as `"ab"` followed by `" x"`, two values. + Case { + input: "(f ```ab```` x`)", + children: &["f", "```ab```", "` x`"], + janet_value: r#""ab" then " x""#, + }, + // Backtick is not a symbol character, so it ends the token before it + // and opens a literal with no whitespace in between. + Case { + input: "(foo`bar`)", + children: &["foo", "`bar`"], + janet_value: r#"(foo "bar")"#, + }, + Case { + input: "(`bar`foo)", + children: &["`bar`", "foo"], + janet_value: r#"("bar" foo)"#, + }, + // `@` before one is Janet's mutable buffer literal: the same lexical + // extent with a different runtime type. `@` is already a reader prefix + // here, so it stays glued to the literal instead of scanning loose. + Case { + input: "(f @```abc```)", + children: &["f", "@```abc```"], + janet_value: r#"@"abc""#, + }, + ]; + + for case in cases { + let tree = SyntaxTree::parse_with_dialect(case.input, Dialect::Janet) + .unwrap_or_else(|error| panic!("{}: {error}", case.input)); + let form = &tree.root_view().children[0]; + let children = form + .children + .iter() + .map(|child| child.span.slice(case.input)) + .collect::>(); + assert_eq!( + children, case.children, + "{} (janet reads {})", + case.input, case.janet_value + ); + } +} + +/// An unterminated long string is refused, not read to EOF as one giant atom. +/// +/// Janet refuses it too: `janet_parser_eof` finds the `longstring` state still +/// on the stack and reports "unexpected end of source". Reading the opener as +/// an atom instead would hand every later command a tree in which the rest of +/// the file is one enormous symbol -- silent corruption of exactly the kind +/// this arm exists to remove -- so it fails loudly. +/// +/// Note the middle two cases: because the opener is the *whole* run of +/// backticks, ```` `` ```` is a two-backtick opener with no close rather than +/// an empty string, so an empty long string cannot be written at all. Janet +/// agrees, reporting "unexpected end of source, `` opened at line 1". +#[test] +fn janet_unterminated_long_string_is_refused() { + for input in ["(f ```abc)", "(f ``)", "(f ````)", "(f `abc)"] { + let error = SyntaxTree::parse_with_dialect(input, Dialect::Janet) + .expect_err("an unterminated long string must not parse"); + assert!( + matches!(error, ParseError::UnterminatedString(_)), + "{input}: {error:?}" + ); + } +} + +/// The extent rule itself, tested on the one function both parser paths call. +/// +/// The recording path (`atom_long_string_with_prefixes`) and the discarded-form +/// scanner (`skip_form`) each ask `long_string_extent` where the literal ends, +/// which is what stops them disagreeing. That sharing cannot be exercised +/// end-to-end today because Janet has no datum comment -- `#` is always a line +/// comment there, so nothing reaches `skip_form` under this dialect -- so the +/// shared decision is pinned directly instead. +#[test] +fn janet_long_string_extent_matches_janets_reader() { + let policy = DialectReaderPolicy::new(Dialect::Janet); + let closed = |width| Some(LongStringExtent::Closed { width }); + + // Opener run length determines the required close. + assert_eq!(policy.long_string_extent(b"`abc`", 0), closed(5)); + assert_eq!(policy.long_string_extent(b"``abc``", 0), closed(7)); + assert_eq!(policy.long_string_extent(b"```abc```", 0), closed(9)); + // Shorter interior runs are content. + assert_eq!(policy.long_string_extent(b"```a`b```", 0), closed(9)); + assert_eq!(policy.long_string_extent(b"``a```b``", 0), closed(5)); + // Exactly N closes, surplus backticks are left for the next datum. + assert_eq!(policy.long_string_extent(b"```ab```` x`", 0), closed(8)); + // Scanning starts at `pos`, not at 0. + assert_eq!(policy.long_string_extent(b"(f `abc`)", 3), closed(5)); + // No close, and an all-backtick run, are both unterminated. + assert_eq!( + policy.long_string_extent(b"```abc", 0), + Some(LongStringExtent::Unterminated) + ); + assert_eq!( + policy.long_string_extent(b"``", 0), + Some(LongStringExtent::Unterminated) + ); + // Not a long string at all. + assert_eq!(policy.long_string_extent(b"abc", 0), None); + assert_eq!(policy.long_string_extent(b"", 0), None); + // And never one outside Janet. + assert_eq!( + DialectReaderPolicy::new(Dialect::CommonLisp).long_string_extent(b"`abc`", 0), + None + ); +} + +/// A backtick keeps meaning quasiquote everywhere else. Janet is the only +/// dialect that may change, and this pins the other nine so a later edit to +/// `has_long_strings` cannot quietly widen. +#[test] +fn backtick_is_still_quasiquote_outside_janet() { + for dialect in [ + Dialect::CommonLisp, + Dialect::EmacsLisp, + Dialect::Scheme, + Dialect::Racket, + Dialect::Clojure, + Dialect::Fennel, + Dialect::Lfe, + Dialect::Hy, + Dialect::Carp, + Dialect::Unknown, + ] { + let input = "(f `(a b))"; + let tree = SyntaxTree::parse_with_dialect(input, dialect) + .unwrap_or_else(|error| panic!("{}: {error}", dialect.label())); + let form = &tree.root_view().children[0]; + let children = form + .children + .iter() + .map(|child| child.span.slice(input)) + .collect::>(); + assert_eq!(children, vec!["f", "`(a b)"], "{}", dialect.label()); + assert_eq!( + form.children[1].reader_prefixes, + vec![ReaderPrefix::Quasiquote], + "{}", + dialect.label() + ); + } +}