Tool dialect unify

crates/tinytools-agent/src/parse/grammar/bare_json.rs

@@ -16,7 +16,7 @@
 //! alone; `{"name":"Alice","input":"hi"}` is left alone.
 
 use crate::parse::call_object::{AliasPolicy, read_calls};
-use crate::repair::json::{recover_object, strip_code_fence};
+use crate::repair::json::{recover_whole_object, strip_code_fence};
 use crate::types::{CallSource, ParseOptions, ParsedToolCall};
 
 /// The calls in a whole-response JSON value, plus any `content` text it
@@ -36,7 +36,7 @@ pub(crate) fn parse(
         Ok(value) => value,
         // A non-object that parsed strictly is not a call; do not "repair"
         // it into one. Only an object-shaped candidate is worth recovering.
-        Err(_) if first == '{' => recover_object(candidate)?,
+        Err(_) if first == '{' => recover_whole_object(candidate)?,
         Err(_) => return None,
     };
 

crates/tinytools-agent/src/parse/grammar/harmony.rs

@@ -24,6 +24,9 @@ pub(crate) struct Harmony;
 const CHANNEL: &str = "<|channel|>";
 const MESSAGE: &str = "<|message|>";
 const TERMINATORS: &[&str] = &["<|call|>", "<|end|>", "<|return|>"];
+/// The Harmony template's per-turn preamble, always immediately before the
+/// first channel. Furniture, not narrative — see the module doc.
+const START_PREFIX: &str = "<|start|>assistant";
 
 impl Grammar for Harmony {
     fn source(&self) -> CallSource {
@@ -33,10 +36,11 @@ impl Grammar for Harmony {
     fn probe(&self, text: &str, from: usize, _options: &ParseOptions<'_>, mode: ScanMode) -> Probe {
         let mut cursor = from;
         while let Some(idx) = find_ci(text, CHANNEL, cursor) {
+            let start = absorb_start_prefix(text, idx);
             let header_start = idx + CHANNEL.len();
             let Some(message_rel) = find_ci(text, MESSAGE, header_start) else {
                 if mode == ScanMode::Stream {
-                    return Probe::Pending { start: idx };
+                    return Probe::Pending { start };
                 }
                 return Probe::None;
             };
@@ -54,12 +58,17 @@ impl Grammar for Harmony {
                 .min_by_key(|(i, _)| *i);
             let Some((payload_end, term_end)) = terminator else {
                 if mode == ScanMode::Stream {
-                    return Probe::Pending { start: idx };
+                    return Probe::Pending { start };
                 }
                 // Batch: the payload runs to the end of the text.
-                return found(idx, text.len(), &name, after);
+                return found(start, text.len(), &name, after);
             };
-            return found(idx, payload_start + term_end, &name, &after[..payload_end]);
+            return found(
+                start,
+                payload_start + term_end,
+                &name,
+                &after[..payload_end],
+            );
         }
         Probe::None
     }
@@ -69,6 +78,23 @@ impl Grammar for Harmony {
     }
 }
 
+/// Extends a channel marker's position backward over an immediately
+/// preceding [`START_PREFIX`], so it is dropped along with the call instead
+/// of leaking into the narrative (or, mid-stream, being released before the
+/// scrubber knows a call follows it).
+fn absorb_start_prefix(text: &str, idx: usize) -> usize {
+    let Some(prefix_start) = idx.checked_sub(START_PREFIX.len()) else {
+        return idx;
+    };
+    if text.is_char_boundary(prefix_start)
+        && text[prefix_start..idx].eq_ignore_ascii_case(START_PREFIX)
+    {
+        prefix_start
+    } else {
+        idx
+    }
+}
+
 fn found(start: usize, end: usize, name: &str, payload: &str) -> Probe {
     let arguments = recover_object(payload).unwrap_or_else(|| serde_json::json!({}));
     Probe::Found(Block {

crates/tinytools-agent/src/parse/grammar/invoke_xml.rs

@@ -49,6 +49,20 @@ static WRAPPER_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
     .ok()
 });
 
+/// The start of an `<invoke`/`<function` opener, with or without a namespace
+/// or DSML prefix, that does not itself require the tag's `>` to match.
+///
+/// [`OPEN_RE`] only matches a *complete* opening tag, so a fragment boundary
+/// that falls before the `>` — `<atem:invoke name="read"` with nothing
+/// after it yet — makes it match nothing at all. The unprefixed and DSML
+/// spellings are covered by the fixed literal list `probe` also holds on
+/// (`"<invoke "`, `"<function"`, `"<|DSML"`, `"<｜DSML"`), but an XML
+/// namespace prefix is open-ended and cannot be enumerated the same way;
+/// this regex recognizes the tag structurally instead, so a stream split
+/// mid-namespace still holds the fragment back.
+static OPEN_START_RE: LazyLock<Option<Regex>> =
+    LazyLock::new(|| Regex::new(&format!(r"(?is)<{PREFIX}(?:invoke|function)\b")).ok());
+
 /// A closing tag ending an invoke: its own, `</function>`, or a stray
 /// `</tool_call>` some templates substitute.
 static CLOSE_RE: LazyLock<Option<Regex>> = LazyLock::new(|| {
@@ -76,13 +90,19 @@ impl Grammar for InvokeXml {
     }
 
     fn probe(&self, text: &str, from: usize, _options: &ParseOptions<'_>, mode: ScanMode) -> Probe {
-        let pending = pending_opener(
+        let literal_pending = pending_opener(
             text,
             from,
             &["<invoke ", "<function", "<|DSML", "<｜DSML"],
             ">",
             mode,
         );
+        let namespaced_pending =
+            (mode == ScanMode::Stream).then(|| Self::pending_namespaced_open(text, from));
+        let pending = [literal_pending, namespaced_pending.flatten()]
+            .into_iter()
+            .flatten()
+            .min();
         prefer_pending(Self::probe_decided(text, from, mode), pending)
     }
 
@@ -103,6 +123,18 @@ impl Grammar for InvokeXml {
 }
 
 impl InvokeXml {
+    /// The start of a namespaced `<ns:invoke …` (or `<ns:function …`) opener
+    /// whose `>` has not arrived yet, if any.
+    fn pending_namespaced_open(text: &str, from: usize) -> Option<usize> {
+        let re = OPEN_START_RE.as_ref()?;
+        let hay = &text[from..];
+        let m = re.find(hay)?;
+        if hay[m.end()..].contains('>') {
+            return None;
+        }
+        Some(from + m.start())
+    }
+
     /// The next block whose opener is fully present.
     fn probe_decided(text: &str, from: usize, mode: ScanMode) -> Probe {
         let (Some(open_re), Some(wrapper_re), Some(close_re)) =

crates/tinytools-agent/src/parse/grammar/mistral.rs

@@ -50,9 +50,16 @@ impl Grammar for Mistral {
         // v11+: `NAME[ARGS]{…}`, possibly several in a row.
         let mut calls = Vec::new();
         let mut cursor = 0usize;
+        // Set when the loop stopped for a reason a stream fragment boundary
+        // can explain — a name (and possibly a partial `[ARGS]`) not yet
+        // finished, or a complete `NAME[ARGS]` whose JSON body has not fully
+        // arrived — as opposed to text that is definitively not a
+        // continuation (an invalid name).
+        let mut ambiguous_tail = false;
         loop {
             let rest = &after[cursor..];
             let Some(args_rel) = rest.find(ARGS) else {
+                ambiguous_tail = could_be_v11_continuation(rest);
                 break;
             };
             let name = rest[..args_rel].trim();
@@ -65,6 +72,10 @@ impl Grammar for Mistral {
             }
             let payload = &rest[args_rel + ARGS.len()..];
             let Some((value, consumed)) = extract_first_json_value_with_end(payload) else {
+                // A valid name and a complete `[ARGS]` marker, but the JSON
+                // body has not arrived complete yet — streaming cannot tell
+                // that apart from a fragment boundary landing mid-object.
+                ambiguous_tail = true;
                 break;
             };
             let arguments = if value.is_object() {
@@ -78,18 +89,13 @@ impl Grammar for Mistral {
         if !calls.is_empty() {
             // The v11 form allows a second call to follow directly with no
             // fresh `[TOOL_CALLS]` marker (`NAME[ARGS]{…}NAME2[ARGS]{…}`).
-            // If the buffered text ends right where the trailing bytes
-            // could still grow into another such name, a stream fragment
-            // has not necessarily finished the block — finalizing now would
-            // drop the continuation call the moment it arrives split across
-            // a fragment boundary. Hold the whole block until either more
+            // If the buffered text ends right where the trailing bytes are
+            // still ambiguous, a stream fragment has not necessarily
+            // finished the block — finalizing now would drop the
+            // continuation call the moment it arrives split across a
+            // fragment boundary. Hold the whole block until either more
             // text disambiguates it or the stream ends.
-            let could_continue = mode == ScanMode::Stream
-                && after[cursor..]
-                    .trim_start()
-                    .chars()
-                    .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.');
-            if could_continue {
+            if mode == ScanMode::Stream && ambiguous_tail {
                 return Probe::Pending { start };
             }
             return Probe::Found(Block {
@@ -115,3 +121,17 @@ impl Grammar for Mistral {
         &["[TOOL_CALLS]"]
     }
 }
+
+/// Whether `rest` (the text left over after the last complete v11 call, or
+/// the whole body when no call has been read yet) is still consistent with
+/// growing into another `NAME[ARGS]` pair: a run of name characters,
+/// optionally followed by a proper prefix of the `[ARGS]` marker.
+/// `NAME[ARGS]` itself never reaches this check — the caller only calls it
+/// once `rest.find(ARGS)` has already failed.
+fn could_be_v11_continuation(rest: &str) -> bool {
+    let trimmed = rest.trim_start();
+    let name_len = trimmed
+        .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_' || c == '.'))
+        .unwrap_or(trimmed.len());
+    ARGS.starts_with(&trimmed[name_len..])
+}

crates/tinytools-agent/src/parse/grammar/sentinel.rs

@@ -56,26 +56,26 @@ impl Grammar for Sentinel {
     }
 
     fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe {
-        let pending = pending_opener(
-            text,
-            from,
-            &[
-                "<|tool_call",
-                "<｜tool▁call",
-                "<|tool_calls",
-                "<｜tool▁calls",
-            ],
-            ">",
-            mode,
-        );
+        let pending = pending_opener(text, from, self.openers(), ">", mode);
         prefer_pending(Self::probe_decided(text, from, options, mode), pending)
     }
 
     fn openers(&self) -> &'static [&'static str] {
+        // The bar style (`|` / `｜`) and the word separator (`_` / `▁`) vary
+        // independently — `SEP_RE`/`CALL_BEGIN_RE`/etc. accept all four
+        // combinations — so every combination needs its own literal here.
+        // Only two of the four were listed before, which let a split after
+        // e.g. `<｜tool_` (fullwidth bar, ASCII underscore) leak: neither
+        // literal is a substring of it, so `hold_from` released it as plain
+        // text and the completed marker was never recognized.
         &[
             "<|tool_call",
+            "<|tool▁call",
+            "<｜tool_call",
             "<｜tool▁call",
             "<|tool_calls",
+            "<|tool▁calls",
+            "<｜tool_calls",
             "<｜tool▁calls",
         ]
     }

crates/tinytools-agent/src/parse/grammar/tagged.rs

@@ -45,8 +45,19 @@ pub(crate) struct Tagged;
 static TAG_RE: LazyLock<Option<Regex>> =
     LazyLock::new(|| Regex::new(r"(?i)<[|/\s]*tool[_-]?call(?:[|/\s]*|\s+[^>]*)>").ok());
 
-/// Openers a fenced block can carry.
-const FENCE_OPENERS: &[&str] = &["```tool_call", "```toolcall", "```tool-call", "```invoke"];
+/// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed
+/// separately from `` ```tool_call `` rather than relying on a prefix match:
+/// `next_opener` requires the language to end exactly at the literal, so
+/// without its own entry the plural spelling — which
+/// [`crate::parse::protected::TOOL_CALL_LANGUAGES`] already classifies as a
+/// call language, not a protected example — would never be recognized here.
+const FENCE_OPENERS: &[&str] = &[
+    "```tool_call",
+    "```toolcall",
+    "```tool-call",
+    "```tool_calls",
+    "```invoke",
+];
 
 /// Kimi-family argument-quote sentinel that leaks in place of `"`.
 const ARG_QUOTE_SENTINEL: &str = "<|\"|>";
@@ -94,6 +105,7 @@ impl Grammar for Tagged {
             "```tool_call",
             "```toolcall",
             "```tool-call",
+            "```tool_calls",
             "```invoke",
         ]
     }

crates/tinytools-agent/src/parse/test/bare_json.rs

@@ -73,6 +73,12 @@ fn bare_recovery_never_swallows_a_genuine_text_answer() {
         r#"{"name":42}"#,
         r#""just a string""#,
         "[1, 2, 3]",
+        // A damaged leading object followed by unrelated trailing prose (or
+        // another object) must not be recovered via the trailing-noise rung
+        // that `recover_object` allows for marker-delimited call bodies —
+        // bare JSON has no such marker, so the whole response must be the
+        // call.
+        r#"{"name":"shell","arguments":{"command":"x"}} explanation {}"#,
     ] {
         let (cleaned, calls) = parse(text);
         assert!(
@@ -83,6 +89,17 @@ fn bare_recovery_never_swallows_a_genuine_text_answer() {
     }
 }
 
+#[test]
+fn parse_options_default_matches_new_and_allows_bare_json() {
+    // The struct doc says the default allows bare JSON; a derived `Default`
+    // would instead leave `allow_bare_json` at `bool`'s `false`.
+    let outcome = crate::parse::parse_text(
+        r#"{"name":"echo","arguments":{}}"#,
+        &ParseOptions::default(),
+    );
+    assert_eq!(outcome.calls.len(), 1);
+}
+
 #[test]
 fn bare_json_can_be_disabled() {
     let options = ParseOptions::new().without_bare_json();

crates/tinytools-agent/src/parse/test/harmony_mistral.rs

@@ -31,6 +31,17 @@ fn harmony_call_with_start_prefix_and_no_terminator_parses_in_batch() {
     assert_eq!(calls[0].name, "read");
 }
 
+#[test]
+fn harmony_start_prefix_is_consumed_as_furniture() {
+    // `<|start|>assistant` is documented as furniture that precedes the
+    // first channel of a turn; it must not leak into the narrative.
+    let response = "<|start|>assistant<|channel|>commentary to=functions.read<|message|>{\"path\":\"a\"}<|call|>";
+    let (text, calls) = parse(response);
+    assert!(text.is_empty(), "{text:?}");
+    assert_eq!(calls.len(), 1);
+    assert_eq!(calls[0].name, "read");
+}
+
 #[test]
 fn harmony_channel_with_target_but_no_message_is_not_a_call_in_batch_mode() {
     // No `<|message|>` ever arrives, so a batch parse cannot know whether a
@@ -160,6 +171,30 @@ fn mistral_marker_with_no_call_yet_is_held_while_streaming() {
     assert_eq!(second.calls[0].name, "get_weather");
 }
 
+#[test]
+fn mistral_v11_second_call_split_mid_args_bracket_is_not_lost() {
+    // The split falls after the second call's opening `[`, inside the
+    // `[ARGS]` marker itself rather than inside its name — a stream
+    // fragment boundary a naive name-only predicate does not recognize as
+    // still-ambiguous.
+    use crate::stream::StreamScrubber;
+
+    let mut s = StreamScrubber::new();
+    let first = s.feed("[TOOL_CALLS]a[ARGS]{}");
+    assert!(first.calls.is_empty(), "must hold until disambiguated");
+    let second = s.feed("b[");
+    assert!(second.calls.is_empty(), "must still hold: {second:?}");
+    let third = s.feed("ARGS]{}");
+    assert!(
+        third.calls.is_empty(),
+        "the block ends exactly at the fragment boundary, still ambiguous: {third:?}"
+    );
+    let flushed = s.flush();
+    assert_eq!(flushed.calls.len(), 2);
+    assert_eq!(flushed.calls[0].name, "a");
+    assert_eq!(flushed.calls[1].name, "b");
+}
+
 #[test]
 fn mistral_v11_second_call_split_across_fragments_is_not_lost() {
     // The v11 form lets a second call follow directly with no fresh

crates/tinytools-agent/src/parse/test/tagged.rs

@@ -114,6 +114,19 @@ fn fenced_tool_call_block_parses() {
     assert_eq!(calls.len(), 1);
 }
 
+#[test]
+fn fenced_tool_calls_plural_block_parses() {
+    // `protected::TOOL_CALL_LANGUAGES` already classifies `tool_calls`
+    // (plural) as a call language, not a protected code example; this
+    // grammar must recognize it as an opener too, not just decline to
+    // protect it.
+    let markdown = "before\n```tool_calls\n[{\"name\":\"a\",\"arguments\":{}}]\n```\nafter";
+    let (text, calls) = parse(markdown);
+    assert_eq!(text, "before\nafter");
+    assert_eq!(calls.len(), 1);
+    assert_eq!(calls[0].name, "a");
+}
+
 #[test]
 fn fenced_block_closed_by_stray_tag_parses() {
     let hybrid = "```tool_call\n{\"name\":\"echo\",\"arguments\":{}}\n</tool_call>\nrest";