feat(agent): universal tool-call parse/repair/stream/render pipeline

Cargo.toml

@@ -14,7 +14,7 @@ exclude = ["worktrees"]
 # true`, so the version the release workflow bumps is written in exactly one
 # place and every crate moves together.
 [workspace.package]
-version = "0.2.0"
+version = "0.3.0"
 edition = "2024"
 rust-version = "1.88"
 license = "GPL-3.0-only"

crates/tinytools-agent/Cargo.toml

@@ -14,7 +14,7 @@ readme = "README.md"
 regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-tinytools = { path = "../tinytools", version = "0.2.0" }
+tinytools = { path = "../tinytools", version = "0.3.0" }
 tracing = { workspace = true, optional = true }
 
 [features]

crates/tinytools-agent/README.md

@@ -1,18 +1,57 @@
 # TinyTools Agent Protocols
 
-`tinytools-agent` owns the model-facing protocol around a tool declaration:
-tool-call parsing, P-Format signatures, XML and native dialects, catalogue and
-result-block rendering, and safe transcript replay.
+`tinytools-agent` owns the model-facing protocol around a tool declaration,
+**once**, for every consumer: how a model is told to call a tool, how its
+answer is read back — in every surface syntax a model has been seen to use —
+how damaged names and arguments are repaired, how results are rendered, and
+how a transcript is replayed.
 
-It consumes `tinytools::ToolSpec` and never executes a tool. Permission checks,
-approvals, sandboxing, timeouts, provider requests, and progress reporting
-remain responsibilities of the consuming harness or host.
+It consumes `tinytools::ToolSpec` and never executes a tool. Permission
+checks, approvals, sandboxing, timeouts, provider requests, call-id minting,
+and the unknown-tool policy remain the consuming harness's or host's.
 
-The primary API is:
+## Modules
 
-- root parsing and P-Format helpers;
-- `dialect::ToolDialect` and the XML, P-Format, and native implementations;
-- provider-neutral transcript, call, result, and message block types.
+| Module | Owns |
+| --- | --- |
+| `parse` | `parse_text(text, &ParseOptions) -> ParseOutcome`: the scan engine, one grammar per surface syntax, code-fence protection |
+| `repair` | `json::recover_object` (relaxed / damaged JSON), `name::resolve` (damaged tool names against the offered set), `args` (aliases, envelopes, schema-guided coercion) |
+| `stream` | `StreamScrubber`: the same grammars applied to a live text stream, releasing safe text and completed calls as they arrive |
+| `render` | the catalogue, the protocol block for each dialect, the `<tool_result>` envelope and transcript replay |
+| `dialect` | `ToolDialect` binding one rendering to one parser: `XmlDialect`, `PFormatDialect`, `NativeDialect` |
+| `types` | `ParsedToolCall`, `CallSource`, `ParseOptions`, `ParseOutcome`, `ParseDiagnostic` |
+
+## Grammars
+
+| `CallSource` | Shape | Seen from |
+| --- | --- | --- |
+| `TaggedJson` | `<tool_call>{json}</tool_call>`, `<toolcall>`, `<tool-call>`, bare `<invoke>`, attribute form `<tool_call id="…">`, garbled `<\|tool_call>…<tool_call\|>`, `call:` prefix, fenced ```` ```tool_call ````, Kimi `NAME{…}` bodies | Hermes / Qwen templates, OpenRouter, Composio sub-agents, Kimi K2 |
+| `InvokeXml` | `<invoke name><parameter name>`, DeepSeek DSML `<｜DSML｜invoke …>`, namespaced `<atem:invoke>`, `<function=NAME><parameter=k>`, `<function name>` | Claude, DeepSeek V3/V4, muse-spark, Llama / Qwen / Gemma |
+| `Sentinel` | `<｜tool▁call▁begin｜>…<｜tool▁call▁end｜>`, `<\|tool_call_begin\|>…<\|tool_call_end\|>` | DeepSeek R1 / V3, Kimi K2 |
+| `Harmony` | `<\|channel\|>commentary to=NAME<\|message\|>{json}<\|call\|>` | gpt-oss |
+| `Mistral` | `[TOOL_CALLS][{…}]`, `[TOOL_CALLS]NAME[ARGS]{…}` | Mistral |
+| `Glm` | `tool/param>value` lines | GLM |
+| `BareJson` | the whole response is one object / `tool_calls` envelope | Minimax gateways, `llama3.2:3b` under `tool_choice: required` |
+| `PFormat` | `name[0\|value]` inside a tag, registry-gated | any prompted model |
+
+Adding a grammar is one file under `src/parse/grammar/` and one entry in
+`GRAMMARS`; batch parsing, streaming, and every dialect pick it up.
+
+## Bounds
+
+Recovery is forgiving because every accommodation was a real capture, and
+bounded because the alternative is a phantom call:
+
+- a call needs a marker; only the bare-JSON path runs without one, and it
+  requires the entire response to be the value;
+- argument keys are aliased, tool names are not; a bare object needs the
+  canonical `arguments` key or a name the caller offered;
+- names are repaired only to a **unique** offered tool, never invented;
+- a fenced code block with a language is an example, not a call;
+- an open string in truncated JSON is never closed by guessing;
+- this crate never mints call ids.
+
+Diagnostics carry lengths and names, never model output.
 
 The crate has no dependency on TinyAgents or an inference/provider runtime.
 The optional `tracing` feature emits diagnostics for parser recovery and

crates/tinytools-agent/src/dialect/mod.rs

@@ -29,26 +29,25 @@
 //! never decides what is allowed to *happen*. That boundary is what keeps a
 //! host's security policy in the host, where it can be audited.
 
-mod catalogue;
 mod native;
 mod pairing;
 mod pformat;
-mod text;
 mod types;
 mod xml;
 
-pub use catalogue::{CATALOGUE_HEADING, render_json_catalogue, render_pformat_catalogue};
+pub use crate::render::{
+    CATALOGUE_HEADING, TOOL_RESULTS_PREFIX, render_json_catalogue, render_pformat_catalogue,
+};
 pub use native::NativeDialect;
 pub use pairing::pair_tool_cycles;
 pub use pformat::PFormatDialect;
-pub use text::TOOL_RESULTS_PREFIX;
 pub use types::{
     DialectMessage, DialectResponse, DialectRole, NativeToolCall, ToolCallFormat, ToolOutcome,
     ToolResultEntry, TranscriptEntry,
 };
 pub use xml::XmlDialect;
 
-use crate::ParsedToolCall;
+use crate::types::ParsedToolCall;
 use tinytools::ToolSpec;
 
 /// One complete way of speaking tools to a model.

crates/tinytools-agent/src/dialect/native.rs

@@ -17,7 +17,8 @@ use super::types::{
     DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, ToolResultEntry, TranscriptEntry,
 };
 use super::xml::XmlDialect;
-use crate::ParsedToolCall;
+use crate::render;
+use crate::types::ParsedToolCall;
 use tinytools::ToolSpec;
 
 /// Call id used when an outcome carries none. Only reachable if a host hands
@@ -48,9 +49,10 @@ impl ToolDialect for NativeDialect {
         let calls: Vec<ParsedToolCall> = response
             .tool_calls
             .iter()
-            .map(|call| ParsedToolCall {
-                name: call.name.clone(),
-                arguments: match serde_json::from_str::<Value>(&call.arguments) {
+            .map(|call| ParsedToolCall::native(
+                call.id.clone(),
+                call.name.clone(),
+                match serde_json::from_str::<Value>(&call.arguments) {
                     Ok(value @ Value::Object(_)) => value,
                     Ok(other) => {
                         #[cfg(not(feature = "tracing"))]
@@ -73,8 +75,7 @@ impl ToolDialect for NativeDialect {
                         Value::Object(serde_json::Map::new())
                     }
                 },
-                id: Some(call.id.clone()),
-            })
+            ))
             .collect();
 
         if !calls.is_empty() {
@@ -144,15 +145,7 @@ impl ToolDialect for NativeDialect {
         // No catalogue: the provider already has the full schemas in the
         // request. What the model still needs is the behavioural half —
         // notably that narrating an intention is not calling a tool.
-        [
-            "## Tool Use Protocol",
-            "",
-            "When a tool is needed, emit tool calls directly via the model's native tool-calling output.",
-            "Do not only narrate intent (for example, avoid \"Let me check...\") without emitting the tool call.",
-            "After tool results are provided, continue reasoning and then produce the final answer.",
-            "",
-        ]
-        .join("\n")
+        render::native_instructions()
     }
 
     fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec<DialectMessage> {

crates/tinytools-agent/src/dialect/pformat.rs

@@ -15,9 +15,11 @@
 use std::sync::Arc;
 
 use super::ToolDialect;
-use super::text;
 use super::types::{DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, TranscriptEntry};
-use crate::{PFormatRegistry, ParsedToolCall, parse_tool_calls_with_pformat};
+use crate::parse::parse_text;
+use crate::render;
+use crate::types::ParseOptions;
+use crate::{PFormatRegistry, ParsedToolCall};
 use tinytools::ToolSpec;
 
 /// Positional tool calling, driven by a registry of parameter layouts.
@@ -56,52 +58,20 @@ impl PFormatDialect {
     /// The protocol block — **protocol only**, no catalogue.
     ///
     /// The signatures live in the prompt's tool section, rendered by
-    /// [`super::catalogue::render_pformat_catalogue`] from the same schemas
+    /// [`crate::render::render_pformat_catalogue`] from the same schemas
     /// this dialect parses against. Repeating them here is the "tools listed
     /// twice" pattern the JSON dialect is stuck with, and it means adding a
     /// tool changes the prompt in one place instead of two.
     #[must_use]
     pub fn instructions() -> String {
-        let mut instructions = String::new();
-        instructions.push_str("## Tool Use Protocol\n\n");
-        instructions.push_str(
-            "Tool calls use **P-Format** (Parameter-Format): compact, slot-indexed, \
-             pipe-delimited syntax wrapped in `<tool_call>` tags. ~80% cheaper on tokens \
-             than JSON.\n\n",
-        );
-        instructions
-            .push_str("```\n<tool_call>\nget_weather[0|London|1|metric]\n</tool_call>\n```\n\n");
-        instructions.push_str(
-            "**Rules:**\n\
-             - Form: `name[index|value|index|value|...]`. Each value is preceded by the slot \
-               number it fills, taken from that tool's `Call as:` signature in the `## Tools` \
-               section above.\n\
-             - **Send only the arguments you mean to send.** To pass just the third slot, \
-               write `name[2|value]` — there are no empty slots to count.\n\
-             - The signature shows each slot as `index|<name>`, e.g. \
-               `search[0|<query>|1|<limit>]`. `<name>` is a placeholder: replace it with the \
-               value, and do not send the name itself.\n\
-             - Empty calls: `name[]` for zero-arg tools, or for a call sending no arguments.\n\
-             - A call whose indices are missing, non-numeric, or not in the signature is \
-               **rejected** — it will not run. Copy the numbers from the signature.\n\
-             - Escapes inside argument values: `\\|` → `|`, `\\]` → `]`, `\\\\` → `\\`.\n\
-             - You may emit multiple `<tool_call>` blocks in a single response. Each tag holds \
-               exactly one call.\n\
-             - After tool execution, results appear in `<tool_result>` tags. Continue reasoning \
-               with the results until you can give a final answer.\n\
-             - If you genuinely need a complex nested argument that p-format can't express, \
-               you may fall back to the JSON form: \
-               `<tool_call>{\"name\":\"...\",\"arguments\":{...}}</tool_call>`. Prefer p-format \
-               for everything else.\n\n",
-        );
-        instructions
+        render::pformat_instructions()
     }
 }
 
 impl ToolDialect for PFormatDialect {
     fn parse_response(&self, response: &DialectResponse) -> (String, Vec<ParsedToolCall>) {
-        let (text, calls) =
-            parse_tool_calls_with_pformat(response.text_or_empty(), self.registry.as_ref());
+        let options = ParseOptions::new().with_registry(self.registry.as_ref());
+        let (text, calls) = parse_text(response.text_or_empty(), &options).into_parts();
         crate::telemetry::debug!(
             parse_mode = "pformat_combined",
             parsed_tool_calls = calls.len(),
@@ -111,15 +81,15 @@ impl ToolDialect for PFormatDialect {
     }
 
     fn format_results(&self, results: &[ToolOutcome]) -> Vec<TranscriptEntry> {
-        text::format_results(results)
+        render::format_results(results)
     }
 
     fn prompt_instructions(&self, _tools: &[ToolSpec]) -> String {
         Self::instructions()
     }
 
     fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec<DialectMessage> {
-        text::to_provider_messages(history)
+        render::to_provider_messages(history)
     }
 
     fn should_send_tool_specs(&self) -> bool {

crates/tinytools-agent/src/dialect/test.rs

@@ -822,3 +822,16 @@ fn the_native_dialect_carries_the_flag_onto_the_entry() {
     assert!(results[0].trusted_verbatim);
     assert_eq!(results[0].content, "payload");
 }
+
+#[test]
+fn json_call_rendering_round_trips_through_the_parser() {
+    let rendered = crate::render::render_json_calls([
+        ("read_file", &serde_json::json!({ "path": "a.txt" })),
+        ("noargs", &serde_json::json!({})),
+    ]);
+    let (_, calls) = crate::parse_tool_calls(&rendered);
+    assert_eq!(calls.len(), 2);
+    assert_eq!(calls[0].name, "read_file");
+    assert_eq!(calls[0].arguments["path"], "a.txt");
+    assert_eq!(calls[1].arguments, serde_json::json!({}));
+}

crates/tinytools-agent/src/dialect/xml.rs

@@ -5,47 +5,37 @@
 //! spells out its argument names, and the catalogue carries full schemas — and
 //! it is the one that works everywhere, which is why it stays the fallback
 //! rather than being retired.
+//!
+//! Parsing is not limited to the advertised form. A model told to write
+//! `<tool_call>` may answer in whatever its template prefers, so the response
+//! goes through every grammar in [`crate::parse`].
 
 use super::ToolDialect;
-use super::catalogue::render_json_catalogue;
-use super::text;
 use super::types::{DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, TranscriptEntry};
-use crate::{ParsedToolCall, parse_tool_calls};
+use crate::parse::parse_text;
+use crate::render;
+use crate::types::{ParseOptions, ParsedToolCall};
 use tinytools::ToolSpec;
 
 /// JSON-in-tag tool calling.
 #[derive(Debug, Default, Clone, Copy)]
 pub struct XmlDialect;
 
 impl XmlDialect {
-    /// Recover tool calls from raw model text.
+    /// Recover tool calls from raw model text with default options.
     ///
     /// Shared with the other two dialects: p-format falls back to it per tag,
     /// and the native dialect uses it to recover calls a model narrated as text
     /// despite having a structured channel available.
     #[must_use]
     pub fn parse_text(text: &str) -> (String, Vec<ParsedToolCall>) {
-        parse_tool_calls(text)
+        parse_text(text, &ParseOptions::new()).into_parts()
     }
 
     /// The protocol block plus the full-schema catalogue.
-    ///
-    /// This dialect embeds its own catalogue rather than leaving it to the
-    /// prompt's tool section, because the schemas it needs are the protocol:
-    /// a model writing `{"arguments": {…}}` by hand has to know the argument
-    /// names, and there is nowhere else in the prompt that tells it.
     #[must_use]
     pub fn instructions(tools: &[ToolSpec]) -> String {
-        let mut instructions = String::new();
-        instructions.push_str("## Tool Use Protocol\n\n");
-        instructions
-            .push_str("To use a tool, wrap a JSON object in <tool_call></tool_call> tags:\n\n");
-        instructions.push_str(
-            "```\n<tool_call>\n{\"name\": \"tool_name\", \"arguments\": {\"param\": \"value\"}}\n</tool_call>\n```\n\n",
-        );
-        instructions.push_str("### Available Tools\n\n");
-        instructions.push_str(&render_json_catalogue(tools));
-        instructions
+        render::json_instructions(tools)
     }
 }
 
@@ -61,15 +51,15 @@ impl ToolDialect for XmlDialect {
     }
 
     fn format_results(&self, results: &[ToolOutcome]) -> Vec<TranscriptEntry> {
-        text::format_results(results)
+        render::format_results(results)
     }
 
     fn prompt_instructions(&self, tools: &[ToolSpec]) -> String {
         Self::instructions(tools)
     }
 
     fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec<DialectMessage> {
-        text::to_provider_messages(history)
+        render::to_provider_messages(history)
     }
 
     fn should_send_tool_specs(&self) -> bool {