fix(syntax): implement Carp's reader macros and literals

packages/core/syntax/src/sexpr/reader.rs

@@ -51,13 +51,24 @@ pub fn apply_reader_prefix_context(
             // binary segment is `(value (size N) (unit 8))` and a map literal's
             // values are arbitrary forms, so rename and reference tracking must
             // keep descending into them.
+            //
+            // Carp's `&x`/`@x`/`~x`/`$[...]` join them for the same reason and
+            // more directly: each wraps an ordinary subexpression -- `(ref x)`,
+            // `(copy x)`, a deref, a static array literal -- so `x` is a live
+            // reference that rename and reference tracking must still see.
+            // Treating them as opaque would hide roughly a fifth of every Carp
+            // file from those commands.
             ReaderPrefix::HashLiteral
             | ReaderPrefix::LfeBinary
             | ReaderPrefix::LfeMap
             | ReaderPrefix::LfeStruct
             | ReaderPrefix::Metadata
             | ReaderPrefix::ReaderConditional
-            | ReaderPrefix::ReaderConditionalSplicing => {}
+            | ReaderPrefix::ReaderConditionalSplicing
+            | ReaderPrefix::Ref
+            | ReaderPrefix::Copy
+            | ReaderPrefix::Deref
+            | ReaderPrefix::StaticArray => {}
         }
     }
 

packages/core/syntax/src/sexpr/reader_policy.rs

@@ -111,8 +111,16 @@ impl DialectReaderPolicy {
         }
     }
 
+    /// Whether `byte` separates tokens rather than belonging to one.
+    ///
+    /// Carp joins Clojure in treating a comma as whitespace: upstream's
+    /// `emptyCharacters` is `[space, tab, comma, linebreak, eof, comment]`
+    /// (`src/Parsing.hs`), so a comma is a separator there exactly as it is in
+    /// Clojure. Without this it fell through to the shared quote-prefix table
+    /// and read as an unquote, which is a reader macro Carp does not have.
     pub(super) const fn is_whitespace(self, byte: u8) -> bool {
-        byte.is_ascii_whitespace() || matches!(self.dialect, Dialect::Clojure) && byte == b','
+        byte.is_ascii_whitespace()
+            || matches!(self.dialect, Dialect::Clojure | Dialect::Carp) && byte == b','
     }
 
     pub(super) fn line_comment_width(self, bytes: &[u8], pos: usize) -> Option<usize> {
@@ -392,6 +400,19 @@ impl DialectReaderPolicy {
                 2
             }
             Dialect::Clojure if byte == b'\\' => 1,
+            // Carp spells a character literal `\a`, like Clojure. Upstream's
+            // `aChar` is `Parsec.char '\\'` followed by one of the named
+            // characters (`space`, `newline`, `tab`, `backspace`, `return`,
+            // `formfeed`), `\"`, `\uXXXX`, or `Parsec.anyChar` -- so the
+            // payload is always at least one character and never a delimiter
+            // boundary. Without this arm `\{`, `\}`, `\(`, `\)`, `\[`, `\]`
+            // and `\"` had their payload read as a *real* delimiter, which is
+            // what made `core/Format.carp` and `examples/json_parser.carp`
+            // fail to parse outright. The named spellings need no arm of their
+            // own: consuming `\` plus one character leaves `pace` of `\space`
+            // to the atom scanner, which stops at the same boundary and yields
+            // the one atom `\space`.
+            Dialect::Carp if byte == b'\\' => 1,
             Dialect::EmacsLisp if byte == b'?' && next == Some(b'\\') => 2,
             Dialect::EmacsLisp if byte == b'?' => 1,
             _ => return None,
@@ -429,9 +450,10 @@ impl DialectReaderPolicy {
         let third = bytes.get(pos + 2).copied();
 
         match self.dialect {
-            Dialect::Unknown | Dialect::Carp => self.classify_legacy(byte, next, third),
+            Dialect::Unknown => self.classify_legacy(byte, next, third),
             Dialect::Lfe => self.classify_lfe(bytes, pos),
             Dialect::Hy => self.classify_hy(byte, next, third),
+            Dialect::Carp => Self::classify_carp(byte, next),
             Dialect::CommonLisp => self.classify_common_lisp(bytes, pos),
             Dialect::EmacsLisp => self.classify_emacs_lisp(bytes, pos),
             Dialect::Scheme | Dialect::Racket => self.classify_scheme(bytes, pos),
@@ -617,6 +639,96 @@ impl DialectReaderPolicy {
         }
     }
 
+    /// Carp's reader macros, from the `sexpr` dispatch in `src/Parsing.hs`.
+    ///
+    /// Carp used to share [`Self::classify_legacy`] with LFE, Hy and the
+    /// permissive reader, and the two grammars have almost nothing in common.
+    /// Upstream dispatches on a single lookahead character:
+    ///
+    /// ```text
+    /// sexpr = do
+    ///   c <- Parsec.lookAhead Parsec.anyChar
+    ///   x <- case c of
+    ///     '&' -> ref
+    ///     '~' -> deref
+    ///     '@' -> copy
+    ///     '\'' -> quote
+    ///     '`' -> quasiquote
+    ///     '%' -> Parsec.try unquoteSplicing <|> unquote
+    ///     '(' -> list
+    ///     '[' -> array
+    ///     '$' -> staticArray
+    ///     '{' -> dictionary
+    ///     _ -> atom
+    /// ```
+    ///
+    /// where `readerMacro` consumes the sigil and then recurses into `sexpr`,
+    /// so a sigil prefixes *any* following form -- symbol, list, string
+    /// literal, or another prefixed form (`@@x`, `&@x`). None of `& ~ @ % $ #`
+    /// is in upstream's `validCharacters`, so none can occur inside a symbol
+    /// and each is unambiguously a prefix wherever it appears.
+    ///
+    /// Missing these cost more than tidiness. `&` and `@` alone accounted for
+    /// 1493 bare sigil atoms across 116 of the 248 files in `carp-lang/Carp`,
+    /// each one an extra sibling that inflated its enclosing call's arity, so
+    /// no argument-counting analysis was sound for Carp.
+    ///
+    /// Two upstream forms are deliberately still not implemented here:
+    ///
+    /// * `%` / `%@` (unquote, unquote-splicing). Recognizing them would make
+    ///   the interior of every `` ` `` template read as code rather than data,
+    ///   which *adds* lint findings on macro bodies. That direction needs a
+    ///   per-dialect false-positive audit of its own, and leaving it reads
+    ///   templates as inert data -- the suppressing, safe direction.
+    /// * `{...}` dictionaries desugar to `(Map.from-array [(Pair.init k v)…])`
+    ///   in upstream's reader. Structurally they are already a brace list here,
+    ///   which is the right shape for a structural tool; the desugaring is a
+    ///   semantic concern.
+    const fn classify_carp(byte: u8, next: Option<u8>) -> Option<ReaderMacro> {
+        match byte {
+            // A trailing `\` is a truncated character literal, not a symbol.
+            // Same contract `classify_clojure`, `classify_scheme` and
+            // `classify_emacs_lisp` already state for their own spellings: the
+            // formatter appends a trailing newline, a `\` left as an atom
+            // claims that newline as its character on the next parse, and
+            // `format(format(x))` stops converging. The recorded fuzz input
+            // `fuzz/corpus/format_idempotence/truncated-clojure-char` is
+            // exactly this byte and found it here too.
+            b'\\' if next.is_none() => Some(ReaderMacro::UnsupportedDispatch { width: 1 }),
+            b'&' => prefix(ReaderPrefix::Ref, 1),
+            b'@' => prefix(ReaderPrefix::Copy, 1),
+            b'~' => prefix(ReaderPrefix::Deref, 1),
+            b'\'' => prefix(ReaderPrefix::Quote, 1),
+            b'`' => prefix(ReaderPrefix::Quasiquote, 1),
+            // `staticArray` matches the two-byte string `$[`; a `$` anywhere
+            // else is a parse error upstream, and reading it as an atom here
+            // is the more permissive of the two options.
+            b'$' if matches!(next, Some(b'[')) => prefix(ReaderPrefix::StaticArray, 1),
+            // `#"…"` is a `Pattern` literal and the only `#` form Carp has.
+            // One dispatch byte plus exactly one datum, which is how
+            // `classify_clojure` reads its `#"…"` regex literal too, so both
+            // go through the same string scanner. Upstream's
+            // `parseInternalPattern` accepts a `"` only as `\"` and every
+            // other escape as backslash-plus-one, so a backslash-aware scan
+            // ends the literal on exactly the byte upstream ends it on.
+            b'#' if matches!(next, Some(b'"')) => Some(ReaderMacro::MultiDatum {
+                width: 1,
+                payload_forms: 1,
+            }),
+            // `#"` is the *only* `#` form Carp has. `#` is absent from
+            // upstream's `validCharacters`, so it cannot occur inside a symbol
+            // either, and `atom` has no branch that accepts it: `#` anywhere
+            // else is a parse error upstream. Refusing it here keeps that,
+            // and keeps the robustness suite's out-of-band oracle -- "no
+            // complete document contains a bare `#+`/`#-` standing alone as
+            // its own datum" -- true for Carp rather than exempting it.
+            // Carp inherited `#;`, `#_`, `#+`, `#-`, `#.`, `#'`, `#?`, `#(`,
+            // `#[` and `#{` from the legacy reader; it has none of them.
+            b'#' => Some(ReaderMacro::UnsupportedDispatch { width: 1 }),
+            _ => None,
+        }
+    }
+
     fn classify_common_lisp(self, bytes: &[u8], pos: usize) -> Option<ReaderMacro> {
         let byte = *bytes.get(pos)?;
         let next = bytes.get(pos + 1).copied();

packages/core/syntax/src/sexpr/tests/parser.rs

@@ -2478,3 +2478,150 @@ fn hy_discarded_forms_use_the_same_string_extent() {
         assert_eq!(children, expected, "{input}");
     }
 }
+
+// ---------------------------------------------------------------------------
+// Carp. Every expectation below is read off the `sexpr` dispatch and the
+// `aChar` / `pat` / `emptyCharacters` productions in Carp's own
+// `src/Parsing.hs`, not off this reader's previous behaviour: Carp shared the
+// permissive legacy reader until now, and that reader implemented none of it.
+// ---------------------------------------------------------------------------
+
+/// `&`, `@` and `~` prefix the *following form*, not just a symbol.
+///
+/// This is the defect these tests exist for. `readerMacro` consumes the sigil
+/// and recurses into `sexpr`, so `@(g y)` is one form. Reading it as a bare
+/// `@` atom plus a sibling inflated the enclosing call's arity by one, which
+/// happened 1493 times across 116 of the 248 files in `carp-lang/Carp`.
+#[test]
+fn carp_sigils_prefix_the_following_form() {
+    for (input, prefix, expected) in [
+        ("(f @(g y))", ReaderPrefix::Copy, "@(g y)"),
+        ("(f &(g y))", ReaderPrefix::Ref, "&(g y)"),
+        ("(f ~(g y))", ReaderPrefix::Deref, "~(g y)"),
+        ("(f @x)", ReaderPrefix::Copy, "@x"),
+        ("(f &x)", ReaderPrefix::Ref, "&x"),
+        ("(f $[1 2])", ReaderPrefix::StaticArray, "$[1 2]"),
+    ] {
+        let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp)
+            .unwrap_or_else(|error| panic!("{input}: {error}"));
+        let form = &tree.root_view().children[0];
+        assert_eq!(form.children.len(), 2, "{input}");
+        assert_eq!(form.children[1].span.slice(input), expected, "{input}");
+        assert_eq!(form.children[1].reader_prefixes, vec![prefix], "{input}");
+    }
+}
+
+/// Sigils stack, because `readerMacro` recurses into `sexpr` rather than into
+/// `atom`: `@@x` is `(copy (copy x))` and `&@x` is `(ref (copy x))`.
+#[test]
+fn carp_sigils_stack_in_source_order() {
+    let input = "(f &@x)";
+    let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp).expect("valid");
+    let form = &tree.root_view().children[0];
+    assert_eq!(
+        form.children[1].reader_prefixes,
+        vec![ReaderPrefix::Ref, ReaderPrefix::Copy]
+    );
+}
+
+/// `@"a b"` is a copy of a *string literal*, so the string stays whole.
+///
+/// Because `@` used to glue onto the following token, the opening quote was
+/// swallowed and `(f @"a b")` silently became the two atoms `@"a` and `b"`.
+#[test]
+fn carp_copy_prefix_keeps_a_string_literal_whole() {
+    let input = r#"(f @"a b")"#;
+    let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp).expect("valid");
+    let form = &tree.root_view().children[0];
+    assert_eq!(form.children.len(), 2);
+    assert_eq!(form.children[1].span.slice(input), r#"@"a b""#);
+}
+
+/// `aChar` is `\` plus one character, and the character may be a delimiter.
+///
+/// `\{` is what made `core/Format.carp` fail to parse: the brace was read as a
+/// real delimiter and unbalanced the file. `\ ` is a space character literal
+/// and occurs in `core/String.carp`; `\space` needs no separate arm because
+/// the atom scanner carries the remaining letters into the same atom.
+#[test]
+fn carp_character_literals_cover_delimiters_and_named_characters() {
+    for (input, expected) in [
+        ("(f \\a)", "\\a"),
+        ("(f \\{)", "\\{"),
+        ("(f \\})", "\\}"),
+        ("(f \\()", "\\("),
+        ("(f \\))", "\\)"),
+        ("(f \\[)", "\\["),
+        ("(f \\])", "\\]"),
+        ("(f \\\")", "\\\""),
+        ("(f \\space)", "\\space"),
+        ("(f \\ )", "\\ "),
+    ] {
+        let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp)
+            .unwrap_or_else(|error| panic!("{input}: {error}"));
+        let form = &tree.root_view().children[0];
+        assert_eq!(form.children.len(), 2, "{input}");
+        assert_eq!(form.children[1].span.slice(input), expected, "{input}");
+    }
+}
+
+/// `#"…"` is a `Pattern` literal: one dispatch byte and one datum.
+///
+/// `parseInternalPattern` admits a `"` only as `\"`, so a backslash-aware scan
+/// ends the literal where Carp ends it.
+#[test]
+fn carp_pattern_literal_is_one_span() {
+    for (input, expected) in [
+        (r#"(f #"[a-z]+")"#, r#"#"[a-z]+""#),
+        (r#"(f #"a\"b")"#, r#"#"a\"b""#),
+    ] {
+        let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp)
+            .unwrap_or_else(|error| panic!("{input}: {error}"));
+        let form = &tree.root_view().children[0];
+        assert_eq!(form.children.len(), 2, "{input}");
+        assert_eq!(form.children[1].span.slice(input), expected, "{input}");
+    }
+}
+
+/// A comma is whitespace, not an unquote: `emptyCharacters` lists it beside
+/// space and tab. It separates `deftype` fields and `defn` parameters
+/// throughout `core/`, e.g. `(deftype Point [x Int, y Int])`.
+#[test]
+fn carp_comma_is_whitespace() {
+    let input = "(deftype Point [x Int, y Int])";
+    let tree = SyntaxTree::parse_with_dialect(input, Dialect::Carp).expect("valid");
+    let fields = &tree.root_view().children[0].children[2];
+    assert_eq!(fields.children.len(), 4);
+    assert!(
+        fields
+            .children
+            .iter()
+            .all(|child| child.reader_prefixes.is_empty())
+    );
+}
+
+/// Forms Carp's reader has no dispatch for are refused rather than read as
+/// something else.
+///
+/// `#` is absent from `validCharacters` and `atom` has no branch that accepts
+/// it, so every `#` form except `#"…"` is a read error upstream -- including
+/// the `#;`, `#_`, `#+`, `#-`, `#.`, `#'`, `#?`, `#(`, `#[` and `#{` Carp
+/// inherited from the legacy reader. An unterminated `#"` or `@"` must fail
+/// loudly rather than swallow the rest of the file.
+#[test]
+fn carp_refuses_what_its_reader_has_no_dispatch_for() {
+    for input in [
+        "(f #(g))",
+        "(f #{1})",
+        "#+sbcl (f)",
+        "#;(f)",
+        "(f #\"abc)",
+        "(f @\"abc)",
+        "\\",
+    ] {
+        assert!(
+            SyntaxTree::parse_with_dialect(input, Dialect::Carp).is_err(),
+            "{input} should be refused"
+        );
+    }
+}

packages/core/syntax/src/sexpr/tree.rs

@@ -234,6 +234,22 @@ pub enum ReaderPrefix {
     ReaderConditional,
     /// Clojure splicing reader conditional (`#?@(:clj [a] :cljs [b])`).
     ReaderConditionalSplicing,
+    /// Carp's `&x`, which its reader expands to `(ref x)`.
+    ///
+    /// Carp gets its own variants rather than borrowing [`Self::Function`] --
+    /// the "catch-all single-prefix slot" `quote_edit` describes, which already
+    /// carries Clojure's `@x` and Janet's `|x`. That slot spells itself `#'`
+    /// through [`Self::as_source`], and `as_source` is not decorative: three
+    /// re-emission paths write it straight back into source text. Carp is the
+    /// dialect where three distinct sigils would have collided in that one
+    /// slot, so each keeps its own spelling.
+    Ref,
+    /// Carp's `@x`, which its reader expands to `(copy x)`.
+    Copy,
+    /// Carp's `~x`, which its reader expands to a `Deref` node.
+    Deref,
+    /// Carp's `$[1 2 3]` static array literal, whose `$` glues to the `[`.
+    StaticArray,
 }
 
 impl ReaderPrefix {
@@ -261,6 +277,10 @@ impl ReaderPrefix {
             Self::Metadata => "^",
             Self::ReaderConditional => "#?",
             Self::ReaderConditionalSplicing => "#?@",
+            Self::Ref => "&",
+            Self::Copy => "@",
+            Self::Deref => "~",
+            Self::StaticArray => "$",
         }
     }
 

packages/feature/inline/src/inline_function/domain/macro_expansion/mod.rs

@@ -261,13 +261,21 @@ fn render_prefixed_expression(
         // writes upper case, so this normalises the spelling of a literal
         // rather than changing it. Every span-based path -- the formatter and
         // all structural edits -- keeps the original bytes.
+        //
+        // Carp's `&`/`@`/`~`/`$` join these: each re-emits its own spelling and
+        // keeps expanding the form underneath, which is why they carry a real
+        // `as_source` rather than borrowing another variant's.
         ReaderPrefix::HashLiteral
         | ReaderPrefix::LfeBinary
         | ReaderPrefix::LfeMap
         | ReaderPrefix::LfeStruct
         | ReaderPrefix::Metadata
         | ReaderPrefix::ReaderConditional
-        | ReaderPrefix::ReaderConditionalSplicing => Ok(format!(
+        | ReaderPrefix::ReaderConditionalSplicing
+        | ReaderPrefix::Ref
+        | ReaderPrefix::Copy
+        | ReaderPrefix::Deref
+        | ReaderPrefix::StaticArray => Ok(format!(
             "{}{}",
             prefix.as_source(),
             render_prefixed_expression(