fix(syntax): implement Janet backtick long strings

packages/core/syntax/src/sexpr/parser.rs

@@ -2,7 +2,7 @@ use thiserror::Error;
 
 use crate::dialect::Dialect;
 
-use super::reader_policy::{DialectReaderPolicy, ReaderMacro};
+use super::reader_policy::{DialectReaderPolicy, LongStringExtent, ReaderMacro};
 use super::tree::{Comment, Node, NodeKind, ReaderPrefix, ReaderPrefixes, SyntaxTree};
 use super::types::{ByteOffset, ByteSpan, Delimiter, NodeId};
 
@@ -408,6 +408,35 @@ impl<'a> Parser<'a> {
         Err(ParseError::UnterminatedString(start.get()))
     }
 
+    fn atom_long_string_with_prefixes(
+        &mut self,
+        prefixes: Vec<PrefixToken>,
+    ) -> std::result::Result<(), ParseError> {
+        let start = self.pos;
+        let width = self.long_string_width(start.get())?;
+        self.advance_by(width);
+        self.push_atom(prefixes, start, self.pos);
+        Ok(())
+    }
+
+    /// Byte width of the Janet long string opening at `start`.
+    ///
+    /// An unterminated one is refused rather than read to EOF. Janet refuses
+    /// it too: `janet_parser_eof` finds the `longstring` state still on the
+    /// stack and reports "unexpected end of source". Reading it as an atom
+    /// instead would hand every later command a tree in which the remainder
+    /// of the file is one giant symbol -- silent corruption of exactly the
+    /// kind this fix exists to remove -- so it fails loudly, and reuses the
+    /// error the `"..."` path already raises for the same shape of mistake.
+    fn long_string_width(&self, start: usize) -> std::result::Result<usize, ParseError> {
+        match self.policy.long_string_extent(self.bytes, start) {
+            Some(LongStringExtent::Closed { width }) => Ok(width),
+            Some(LongStringExtent::Unterminated) | None => {
+                Err(ParseError::UnterminatedString(start))
+            }
+        }
+    }
+
     fn atom_with_prefixes(
         &mut self,
         prefixes: Vec<PrefixToken>,
@@ -629,6 +658,9 @@ impl<'a> Parser<'a> {
                 return Err(self.raw_delimiter_error());
             }
             b'"' => self.atom_string_with_prefixes(prefixes)?,
+            b'`' if self.policy.has_long_strings() => {
+                self.atom_long_string_with_prefixes(prefixes)?;
+            }
             _ => self.atom_with_prefixes(prefixes)?,
         }
         Ok(())
@@ -806,6 +838,19 @@ impl<'a> Parser<'a> {
                             return Err(self.raw_delimiter_error());
                         }
                         b'"' => self.skip_string()?,
+                        // Unreachable under Janet today: `#` is always a line
+                        // comment there, so `classify_janet` never returns a
+                        // `Discard` and nothing enters this scanner. It is
+                        // here because `DialectReaderPolicy` exists to stop
+                        // the recording and scanning paths disagreeing about
+                        // how far a reader form reaches, and a Janet datum
+                        // comment added later must not silently reintroduce
+                        // that disagreement. Both paths call the same
+                        // `long_string_extent`, which is what makes them agree.
+                        b'`' if self.policy.has_long_strings() => {
+                            let width = self.long_string_width(self.pos.get())?;
+                            self.advance_by(width);
+                        }
                         _ => self.skip_atom()?,
                     }
                 }

packages/core/syntax/src/sexpr/reader_policy.rs

@@ -21,6 +21,15 @@ pub(super) enum ReaderMacro {
     },
 }
 
+/// How far a Janet long string reaches from its opening backtick run.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum LongStringExtent {
+    /// Total byte width of the literal, both delimiter runs included.
+    Closed { width: usize },
+    /// An opening run with no closing run of the same length before EOF.
+    Unterminated,
+}
+
 /// Dialect-specific lexical decisions shared by normal parsing and discarded
 /// form scanning. Keeping these decisions in one place prevents the two paths
 /// from disagreeing about the extent of a reader form.
@@ -133,14 +142,78 @@ impl DialectReaderPolicy {
         Delimiter::from_open(byte).is_some() || Delimiter::from_close(byte).is_some()
     }
 
+    /// Whether a backtick opens a long string in this dialect.
+    ///
+    /// Janet is the only one. Its `root` state sends every backtick to the
+    /// `longstring` consumer (`src/core/parse.c`), and `symchars` leaves bit
+    /// 0x60 clear, so a backtick is not a symbol character either -- it both
+    /// opens a literal and ends whatever token preceded it.
+    ///
+    /// In every other dialect here a backtick is quasiquote (Common Lisp,
+    /// Scheme, Racket, Emacs Lisp, Fennel, and the permissive legacy reader)
+    /// or syntax-quote (Clojure), which `classify_reader_macro` already
+    /// returns as a one-byte `ReaderPrefix::Quasiquote`. Nothing below may
+    /// change for them.
+    pub(super) const fn has_long_strings(self) -> bool {
+        matches!(self.dialect, Dialect::Janet)
+    }
+
     pub(super) fn is_atom_boundary(self, bytes: &[u8], pos: usize) -> bool {
         bytes.get(pos).is_none_or(|byte| {
             self.is_whitespace(*byte)
                 || Self::is_raw_delimiter(*byte)
+                // Dialect first: `self.dialect` is loop-invariant across the
+                // per-byte calls this makes for every atom in the document, so
+                // for the nine dialects without long strings the test folds
+                // away instead of costing a comparison per byte.
+                || (self.has_long_strings() && *byte == b'`')
                 || self.line_comment_width(bytes, pos).is_some()
         })
     }
 
+    /// How far the Janet long string starting at `pos` reaches, if one starts
+    /// there.
+    ///
+    /// Janet's `longstring` state (`src/core/parse.c`) counts the opening run
+    /// in `argn` while it keeps seeing backticks, then closes the literal on
+    /// the `argn`-th consecutive backtick it meets afterwards. Two consequences
+    /// follow, and both are load-bearing:
+    ///
+    /// * The opener is the *whole* run. ```` ```` ```` is a four-backtick
+    ///   opener, not two empty strings, so an empty long string cannot be
+    ///   written at all.
+    /// * The close is exactly `argn` backticks, not at least `argn`. Janet
+    ///   returns 0 from `stringend` so the character that revealed the end is
+    ///   re-dispatched, which means a longer run leaves its surplus to open
+    ///   the next datum: `` ```ab```` x` `` reads as `"ab"` then `" x"`. A run
+    ///   shorter than `argn` is content ("failed end candidate" pushes the
+    ///   backticks it had buffered back into the string).
+    ///
+    /// There is no escape processing inside one -- the `PFLAG_INSTRING` branch
+    /// has no `\\` case -- and a newline is an ordinary content byte, which is
+    /// the entire point of the form.
+    pub(super) fn long_string_extent(self, bytes: &[u8], pos: usize) -> Option<LongStringExtent> {
+        if !self.has_long_strings() || bytes.get(pos) != Some(&b'`') {
+            return None;
+        }
+        let open_len = backtick_run_length(bytes, pos);
+        let mut cursor = pos + open_len;
+        while cursor < bytes.len() {
+            if bytes[cursor] != b'`' {
+                cursor += 1;
+                continue;
+            }
+            let run = backtick_run_length(bytes, cursor);
+            if run >= open_len {
+                return Some(LongStringExtent::Closed {
+                    width: cursor + open_len - pos,
+                });
+            }
+            cursor += run;
+        }
+        Some(LongStringExtent::Unterminated)
+    }
+
     /// How many bytes introduce a character literal at `pos`, if one starts
     /// there.
     ///
@@ -482,6 +555,14 @@ impl DialectReaderPolicy {
     }
 }
 
+/// How many consecutive backticks start at `pos`.
+fn backtick_run_length(bytes: &[u8], pos: usize) -> usize {
+    bytes[pos..]
+        .iter()
+        .take_while(|byte| **byte == b'`')
+        .count()
+}
+
 /// The Racket language directive, which the reader consumes to end of line.
 pub(crate) const LANG_DIRECTIVE: &str = "#lang";