diff --git a/Cargo.lock b/Cargo.lock index ac95c3c..acb14a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ [[package]] name = "tinytools" -version = "0.2.0" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -160,7 +160,7 @@ dependencies = [ [[package]] name = "tinytools-agent" -version = "0.2.0" +version = "0.3.0" dependencies = [ "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index 8e1903a..c8b237f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ exclude = ["worktrees"] # true`, so the version the release workflow bumps is written in exactly one # place and every crate moves together. [workspace.package] -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.88" license = "GPL-3.0-only" diff --git a/crates/tinytools-agent/Cargo.toml b/crates/tinytools-agent/Cargo.toml index 6795a40..e8f0cf7 100644 --- a/crates/tinytools-agent/Cargo.toml +++ b/crates/tinytools-agent/Cargo.toml @@ -14,7 +14,7 @@ readme = "README.md" regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -tinytools = { path = "../tinytools", version = "0.2.0" } +tinytools = { path = "../tinytools", version = "0.3.0" } tracing = { workspace = true, optional = true } [features] diff --git a/crates/tinytools-agent/README.md b/crates/tinytools-agent/README.md index ee1c328..5943ddd 100644 --- a/crates/tinytools-agent/README.md +++ b/crates/tinytools-agent/README.md @@ -1,18 +1,57 @@ # TinyTools Agent Protocols -`tinytools-agent` owns the model-facing protocol around a tool declaration: -tool-call parsing, P-Format signatures, XML and native dialects, catalogue and -result-block rendering, and safe transcript replay. +`tinytools-agent` owns the model-facing protocol around a tool declaration, +**once**, for every consumer: how a model is told to call a tool, how its +answer is read back — in every surface syntax a model has been seen to use — +how damaged names and arguments are repaired, how results are rendered, and +how a transcript is replayed. -It consumes `tinytools::ToolSpec` and never executes a tool. Permission checks, -approvals, sandboxing, timeouts, provider requests, and progress reporting -remain responsibilities of the consuming harness or host. +It consumes `tinytools::ToolSpec` and never executes a tool. Permission +checks, approvals, sandboxing, timeouts, provider requests, call-id minting, +and the unknown-tool policy remain the consuming harness's or host's. -The primary API is: +## Modules -- root parsing and P-Format helpers; -- `dialect::ToolDialect` and the XML, P-Format, and native implementations; -- provider-neutral transcript, call, result, and message block types. +| Module | Owns | +| --- | --- | +| `parse` | `parse_text(text, &ParseOptions) -> ParseOutcome`: the scan engine, one grammar per surface syntax, code-fence protection | +| `repair` | `json::recover_object` (relaxed / damaged JSON), `name::resolve` (damaged tool names against the offered set), `args` (aliases, envelopes, schema-guided coercion) | +| `stream` | `StreamScrubber`: the same grammars applied to a live text stream, releasing safe text and completed calls as they arrive | +| `render` | the catalogue, the protocol block for each dialect, the `` envelope and transcript replay | +| `dialect` | `ToolDialect` binding one rendering to one parser: `XmlDialect`, `PFormatDialect`, `NativeDialect` | +| `types` | `ParsedToolCall`, `CallSource`, `ParseOptions`, `ParseOutcome`, `ParseDiagnostic` | + +## Grammars + +| `CallSource` | Shape | Seen from | +| --- | --- | --- | +| `TaggedJson` | `{json}`, ``, ``, bare ``, attribute form ``, garbled `<\|tool_call>…`, `call:` prefix, fenced ```` ```tool_call ````, Kimi `NAME{…}` bodies | Hermes / Qwen templates, OpenRouter, Composio sub-agents, Kimi K2 | +| `InvokeXml` | ``, DeepSeek DSML `<｜DSML｜invoke …>`, namespaced ``, ``, `` | Claude, DeepSeek V3/V4, muse-spark, Llama / Qwen / Gemma | +| `Sentinel` | `<｜tool▁call▁begin｜>…<｜tool▁call▁end｜>`, `<\|tool_call_begin\|>…<\|tool_call_end\|>` | DeepSeek R1 / V3, Kimi K2 | +| `Harmony` | `<\|channel\|>commentary to=NAME<\|message\|>{json}<\|call\|>` | gpt-oss | +| `Mistral` | `[TOOL_CALLS][{…}]`, `[TOOL_CALLS]NAME[ARGS]{…}` | Mistral | +| `Glm` | `tool/param>value` lines | GLM | +| `BareJson` | the whole response is one object / `tool_calls` envelope | Minimax gateways, `llama3.2:3b` under `tool_choice: required` | +| `PFormat` | `name[0\|value]` inside a tag, registry-gated | any prompted model | + +Adding a grammar is one file under `src/parse/grammar/` and one entry in +`GRAMMARS`; batch parsing, streaming, and every dialect pick it up. + +## Bounds + +Recovery is forgiving because every accommodation was a real capture, and +bounded because the alternative is a phantom call: + +- a call needs a marker; only the bare-JSON path runs without one, and it + requires the entire response to be the value; +- argument keys are aliased, tool names are not; a bare object needs the + canonical `arguments` key or a name the caller offered; +- names are repaired only to a **unique** offered tool, never invented; +- a fenced code block with a language is an example, not a call; +- an open string in truncated JSON is never closed by guessing; +- this crate never mints call ids. + +Diagnostics carry lengths and names, never model output. The crate has no dependency on TinyAgents or an inference/provider runtime. The optional `tracing` feature emits diagnostics for parser recovery and diff --git a/crates/tinytools-agent/src/dialect/mod.rs b/crates/tinytools-agent/src/dialect/mod.rs index 91773b7..f662ebb 100644 --- a/crates/tinytools-agent/src/dialect/mod.rs +++ b/crates/tinytools-agent/src/dialect/mod.rs @@ -29,26 +29,25 @@ //! never decides what is allowed to *happen*. That boundary is what keeps a //! host's security policy in the host, where it can be audited. -mod catalogue; mod native; mod pairing; mod pformat; -mod text; mod types; mod xml; -pub use catalogue::{CATALOGUE_HEADING, render_json_catalogue, render_pformat_catalogue}; +pub use crate::render::{ + CATALOGUE_HEADING, TOOL_RESULTS_PREFIX, render_json_catalogue, render_pformat_catalogue, +}; pub use native::NativeDialect; pub use pairing::pair_tool_cycles; pub use pformat::PFormatDialect; -pub use text::TOOL_RESULTS_PREFIX; pub use types::{ DialectMessage, DialectResponse, DialectRole, NativeToolCall, ToolCallFormat, ToolOutcome, ToolResultEntry, TranscriptEntry, }; pub use xml::XmlDialect; -use crate::ParsedToolCall; +use crate::types::ParsedToolCall; use tinytools::ToolSpec; /// One complete way of speaking tools to a model. diff --git a/crates/tinytools-agent/src/dialect/native.rs b/crates/tinytools-agent/src/dialect/native.rs index 474f231..589f34c 100644 --- a/crates/tinytools-agent/src/dialect/native.rs +++ b/crates/tinytools-agent/src/dialect/native.rs @@ -17,7 +17,8 @@ use super::types::{ DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, ToolResultEntry, TranscriptEntry, }; use super::xml::XmlDialect; -use crate::ParsedToolCall; +use crate::render; +use crate::types::ParsedToolCall; use tinytools::ToolSpec; /// Call id used when an outcome carries none. Only reachable if a host hands @@ -48,9 +49,10 @@ impl ToolDialect for NativeDialect { let calls: Vec = response .tool_calls .iter() - .map(|call| ParsedToolCall { - name: call.name.clone(), - arguments: match serde_json::from_str::(&call.arguments) { + .map(|call| ParsedToolCall::native( + call.id.clone(), + call.name.clone(), + match serde_json::from_str::(&call.arguments) { Ok(value @ Value::Object(_)) => value, Ok(other) => { #[cfg(not(feature = "tracing"))] @@ -73,8 +75,7 @@ impl ToolDialect for NativeDialect { Value::Object(serde_json::Map::new()) } }, - id: Some(call.id.clone()), - }) + )) .collect(); if !calls.is_empty() { @@ -144,15 +145,7 @@ impl ToolDialect for NativeDialect { // No catalogue: the provider already has the full schemas in the // request. What the model still needs is the behavioural half — // notably that narrating an intention is not calling a tool. - [ - "## Tool Use Protocol", - "", - "When a tool is needed, emit tool calls directly via the model's native tool-calling output.", - "Do not only narrate intent (for example, avoid \"Let me check...\") without emitting the tool call.", - "After tool results are provided, continue reasoning and then produce the final answer.", - "", - ] - .join("\n") + render::native_instructions() } fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec { diff --git a/crates/tinytools-agent/src/dialect/pformat.rs b/crates/tinytools-agent/src/dialect/pformat.rs index 9af67b2..ecd8a74 100644 --- a/crates/tinytools-agent/src/dialect/pformat.rs +++ b/crates/tinytools-agent/src/dialect/pformat.rs @@ -15,9 +15,11 @@ use std::sync::Arc; use super::ToolDialect; -use super::text; use super::types::{DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, TranscriptEntry}; -use crate::{PFormatRegistry, ParsedToolCall, parse_tool_calls_with_pformat}; +use crate::parse::parse_text; +use crate::render; +use crate::types::ParseOptions; +use crate::{PFormatRegistry, ParsedToolCall}; use tinytools::ToolSpec; /// Positional tool calling, driven by a registry of parameter layouts. @@ -56,52 +58,20 @@ impl PFormatDialect { /// The protocol block — **protocol only**, no catalogue. /// /// The signatures live in the prompt's tool section, rendered by - /// [`super::catalogue::render_pformat_catalogue`] from the same schemas + /// [`crate::render::render_pformat_catalogue`] from the same schemas /// this dialect parses against. Repeating them here is the "tools listed /// twice" pattern the JSON dialect is stuck with, and it means adding a /// tool changes the prompt in one place instead of two. #[must_use] pub fn instructions() -> String { - let mut instructions = String::new(); - instructions.push_str("## Tool Use Protocol\n\n"); - instructions.push_str( - "Tool calls use **P-Format** (Parameter-Format): compact, slot-indexed, \ - pipe-delimited syntax wrapped in `` tags. ~80% cheaper on tokens \ - than JSON.\n\n", - ); - instructions - .push_str("```\n\nget_weather[0|London|1|metric]\n\n```\n\n"); - instructions.push_str( - "**Rules:**\n\ - - Form: `name[index|value|index|value|...]`. Each value is preceded by the slot \ - number it fills, taken from that tool's `Call as:` signature in the `## Tools` \ - section above.\n\ - - **Send only the arguments you mean to send.** To pass just the third slot, \ - write `name[2|value]` — there are no empty slots to count.\n\ - - The signature shows each slot as `index|`, e.g. \ - `search[0||1|]`. `` is a placeholder: replace it with the \ - value, and do not send the name itself.\n\ - - Empty calls: `name[]` for zero-arg tools, or for a call sending no arguments.\n\ - - A call whose indices are missing, non-numeric, or not in the signature is \ - **rejected** — it will not run. Copy the numbers from the signature.\n\ - - Escapes inside argument values: `\\|` → `|`, `\\]` → `]`, `\\\\` → `\\`.\n\ - - You may emit multiple `` blocks in a single response. Each tag holds \ - exactly one call.\n\ - - After tool execution, results appear in `` tags. Continue reasoning \ - with the results until you can give a final answer.\n\ - - If you genuinely need a complex nested argument that p-format can't express, \ - you may fall back to the JSON form: \ - `{\"name\":\"...\",\"arguments\":{...}}`. Prefer p-format \ - for everything else.\n\n", - ); - instructions + render::pformat_instructions() } } impl ToolDialect for PFormatDialect { fn parse_response(&self, response: &DialectResponse) -> (String, Vec) { - let (text, calls) = - parse_tool_calls_with_pformat(response.text_or_empty(), self.registry.as_ref()); + let options = ParseOptions::new().with_registry(self.registry.as_ref()); + let (text, calls) = parse_text(response.text_or_empty(), &options).into_parts(); crate::telemetry::debug!( parse_mode = "pformat_combined", parsed_tool_calls = calls.len(), @@ -111,7 +81,7 @@ impl ToolDialect for PFormatDialect { } fn format_results(&self, results: &[ToolOutcome]) -> Vec { - text::format_results(results) + render::format_results(results) } fn prompt_instructions(&self, _tools: &[ToolSpec]) -> String { @@ -119,7 +89,7 @@ impl ToolDialect for PFormatDialect { } fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec { - text::to_provider_messages(history) + render::to_provider_messages(history) } fn should_send_tool_specs(&self) -> bool { diff --git a/crates/tinytools-agent/src/dialect/test.rs b/crates/tinytools-agent/src/dialect/test.rs index d0e6728..5e51eae 100644 --- a/crates/tinytools-agent/src/dialect/test.rs +++ b/crates/tinytools-agent/src/dialect/test.rs @@ -822,3 +822,16 @@ fn the_native_dialect_carries_the_flag_onto_the_entry() { assert!(results[0].trusted_verbatim); assert_eq!(results[0].content, "payload"); } + +#[test] +fn json_call_rendering_round_trips_through_the_parser() { + let rendered = crate::render::render_json_calls([ + ("read_file", &serde_json::json!({ "path": "a.txt" })), + ("noargs", &serde_json::json!({})), + ]); + let (_, calls) = crate::parse_tool_calls(&rendered); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "read_file"); + assert_eq!(calls[0].arguments["path"], "a.txt"); + assert_eq!(calls[1].arguments, serde_json::json!({})); +} diff --git a/crates/tinytools-agent/src/dialect/xml.rs b/crates/tinytools-agent/src/dialect/xml.rs index 15ffaaf..ecbcc41 100644 --- a/crates/tinytools-agent/src/dialect/xml.rs +++ b/crates/tinytools-agent/src/dialect/xml.rs @@ -5,12 +5,16 @@ //! spells out its argument names, and the catalogue carries full schemas — and //! it is the one that works everywhere, which is why it stays the fallback //! rather than being retired. +//! +//! Parsing is not limited to the advertised form. A model told to write +//! `` may answer in whatever its template prefers, so the response +//! goes through every grammar in [`crate::parse`]. use super::ToolDialect; -use super::catalogue::render_json_catalogue; -use super::text; use super::types::{DialectMessage, DialectResponse, ToolCallFormat, ToolOutcome, TranscriptEntry}; -use crate::{ParsedToolCall, parse_tool_calls}; +use crate::parse::parse_text; +use crate::render; +use crate::types::{ParseOptions, ParsedToolCall}; use tinytools::ToolSpec; /// JSON-in-tag tool calling. @@ -18,34 +22,20 @@ use tinytools::ToolSpec; pub struct XmlDialect; impl XmlDialect { - /// Recover tool calls from raw model text. + /// Recover tool calls from raw model text with default options. /// /// Shared with the other two dialects: p-format falls back to it per tag, /// and the native dialect uses it to recover calls a model narrated as text /// despite having a structured channel available. #[must_use] pub fn parse_text(text: &str) -> (String, Vec) { - parse_tool_calls(text) + parse_text(text, &ParseOptions::new()).into_parts() } /// The protocol block plus the full-schema catalogue. - /// - /// This dialect embeds its own catalogue rather than leaving it to the - /// prompt's tool section, because the schemas it needs are the protocol: - /// a model writing `{"arguments": {…}}` by hand has to know the argument - /// names, and there is nowhere else in the prompt that tells it. #[must_use] pub fn instructions(tools: &[ToolSpec]) -> String { - let mut instructions = String::new(); - instructions.push_str("## Tool Use Protocol\n\n"); - instructions - .push_str("To use a tool, wrap a JSON object in tags:\n\n"); - instructions.push_str( - "```\n\n{\"name\": \"tool_name\", \"arguments\": {\"param\": \"value\"}}\n\n```\n\n", - ); - instructions.push_str("### Available Tools\n\n"); - instructions.push_str(&render_json_catalogue(tools)); - instructions + render::json_instructions(tools) } } @@ -61,7 +51,7 @@ impl ToolDialect for XmlDialect { } fn format_results(&self, results: &[ToolOutcome]) -> Vec { - text::format_results(results) + render::format_results(results) } fn prompt_instructions(&self, tools: &[ToolSpec]) -> String { @@ -69,7 +59,7 @@ impl ToolDialect for XmlDialect { } fn to_provider_messages(&self, history: &[TranscriptEntry]) -> Vec { - text::to_provider_messages(history) + render::to_provider_messages(history) } fn should_send_tool_specs(&self) -> bool { diff --git a/crates/tinytools-agent/src/lib.rs b/crates/tinytools-agent/src/lib.rs index 36ac530..835e076 100644 --- a/crates/tinytools-agent/src/lib.rs +++ b/crates/tinytools-agent/src/lib.rs @@ -1,63 +1,46 @@ //! Agent-facing tool-call protocols. //! //! `tinytools` owns the vocabulary a callable tool exposes. This crate owns the -//! model-facing protocol around those declarations: parsing calls, rendering -//! catalogues and result blocks, and replaying tool cycles. -//! -//! A model that supports native tool use hands back structured calls and none -//! of this is needed. Everything else — prompt-guided models, local models, -//! providers whose native mode is unavailable or disabled — emits tool calls as -//! *text*, in whatever shape the model was trained to produce. This module -//! turns that text back into calls. -//! -//! ## Why it is this forgiving -//! -//! Each accommodation here exists because a model actually produced it and the -//! alternative was dropping a well-formed call and burning an agent iteration. -//! Concretely, the parsers accept `` tags in several spellings, -//! fenced `tool_call` blocks, bare JSON objects, Anthropic-style -//! `` XML, and the compact positional -//! P-Format syntax. -//! -//! The permissiveness is bounded on purpose, and the boundary is worth knowing -//! before widening anything: -//! -//! * **Argument keys are aliased; tool names are not.** A model drifting from -//! `arguments` to `args`/`parameters`/`params`/`input` still yields a usable -//! call. The *name* stays strict, because loosening it risks reading a plain -//! JSON answer as a tool call in the whole-response path — turning an ordinary -//! reply into a phantom invocation. -//! * **The generic `input` alias is only honoured behind an explicit marker** -//! (a `tool_calls` array, a `` tag, a fenced block). Untagged text -//! does not get it. -//! * **P-Format refuses to invent argument names for an unknown tool**, so a -//! model cannot tunnel arbitrary JSON through by guessing a tool name that -//! does not exist. +//! model-facing protocol around those declarations — and owns it **once**, for +//! every consumer: +//! +//! * [`parse`] turns model text back into calls, through every surface syntax +//! a model has been seen to use; +//! * [`repair`] recovers damaged JSON, damaged tool names, and mis-shaped +//! arguments after a call has been located; +//! * [`stream`] scrubs the same markup from a live text stream; +//! * [`render`] produces what the model reads: the catalogue, the protocol +//! block, the result envelope; +//! * [`dialect`] binds one rendering to one parser so they cannot drift. +//! +//! A model that supports native tool use hands back structured calls and only +//! [`dialect::NativeDialect`] is involved. Everything else — prompt-guided +//! models, local models, providers whose native mode is unavailable, and +//! native models that narrate a call as text anyway — goes through the rest. //! //! ## What the host still owns //! -//! This module takes **schemas**, never a tool trait object. A host's tool type -//! is its own vocabulary, and depending on it here would defeat the point — so -//! [`pformat::build_registry`] takes `(name, schema)` pairs and the host keeps a -//! one-line adapter over its own tool slice. -//! -//! **Executing** a tool stays host-side: permission checks, sandboxing, -//! approval gates and timeouts are the host's policy and belong where they can -//! be audited. Everything *around* execution — the catalogue the model reads, -//! the results it is shown, the transcript it is replayed — is not -//! host-specific, and lives in [`dialect`]. +//! This crate takes **schemas**, never a tool trait object, and never executes +//! anything. Permission checks, sandboxing, approval gates, timeouts, the +//! unknown-tool policy, and the minting of call ids are the host's, where they +//! can be audited. See [`parse`] for the bounds on how forgiving the parsers +//! are and why. pub mod dialect; -pub(crate) mod parse; +pub mod parse; pub(crate) mod pformat; +pub mod render; +pub mod repair; +pub mod stream; mod telemetry; +pub mod types; + +/// The tool vocabulary this crate renders and parses against, re-exported so +/// a consumer that only speaks the protocol need not name `tinytools` itself. +pub use tinytools; -// The two entry points, plus the building blocks a host legitimately reaches -// for on its own. `extract_json_values` in particular is not a test helper: -// pulling the first JSON object out of model prose is how a host checks a -// required-output contract, which has nothing to do with tool calls. pub use parse::{ - ParsedToolCall, extract_json_values, parse_arguments_value, parse_glm_style_tool_calls, + extract_json_values, parse_arguments_value, parse_glm_style_tool_calls, parse_text, parse_tool_call_value, parse_tool_calls, parse_tool_calls_from_json_value, parse_tool_calls_with_pformat, }; @@ -65,3 +48,5 @@ pub use pformat::{ PFormatParamType, PFormatRegistry, PFormatToolParams, build_registry, parse_call, render_signature, render_signature_from_schema, }; +pub use stream::{StreamScrubber, StreamStep}; +pub use types::{CallSource, ParseDiagnostic, ParseOptions, ParseOutcome, ParsedToolCall}; diff --git a/crates/tinytools-agent/src/parse.rs b/crates/tinytools-agent/src/parse.rs deleted file mode 100644 index 43989ef..0000000 --- a/crates/tinytools-agent/src/parse.rs +++ /dev/null @@ -1,1082 +0,0 @@ -use regex::Regex; -use std::borrow::Cow; -use std::sync::LazyLock; - -#[derive(Debug, Clone)] -/// One model-requested tool invocation recovered from text or structured data. -pub struct ParsedToolCall { - /// Canonical tool name supplied by the model. - pub name: String, - /// Parsed tool arguments, defaulting to an empty object when absent. - pub arguments: serde_json::Value, - /// Provider-assigned call id when the call came from a native - /// tool-use response. `None` for prompt-guided (XML-parsed) - /// tool calls — progress emitters synthesise a fallback id. - pub id: Option, -} - -/// Normalize an argument value, decoding stringified JSON when possible. -#[must_use] -pub fn parse_arguments_value(raw: Option<&serde_json::Value>) -> serde_json::Value { - match raw { - Some(serde_json::Value::String(s)) => serde_json::from_str::(s) - .unwrap_or_else(|_| serde_json::Value::Object(serde_json::Map::new())), - Some(value) => value.clone(), - None => serde_json::Value::Object(serde_json::Map::new()), - } -} - -/// Object keys that may carry the tool **arguments**, in priority order. -/// Models drift from the canonical `arguments` to `args`/`parameters`/etc.; -/// accepting these recovers an otherwise well-formed call (with a correct -/// `name`) instead of dropping it and burning an agent iteration -/// (bug-report-2026-05-26 A3). The tool **name** is deliberately left -/// strict — widening it would risk misreading a plain JSON answer as a -/// tool call in the whole-response parse path. -const TOOL_ARG_KEYS: &[&str] = &["arguments", "args", "parameters", "params", "input"]; - -/// Normalized arguments for the first present key among [`TOOL_ARG_KEYS`] -/// (via [`parse_arguments_value`], which tolerates both stringified and -/// object JSON). Empty-object default when none are present. -fn first_args_by_keys(obj: &serde_json::Value) -> serde_json::Value { - for key in TOOL_ARG_KEYS { - if let Some(v) = obj.get(*key) { - return parse_arguments_value(Some(v)); - } - } - parse_arguments_value(None) -} - -/// Parse a single JSON value as a tool call, honouring the argument-key -/// aliases. -/// -/// The permissive entry point: callers reach a value through an explicit -/// tool-call marker (a `tool_calls` array, a `` tag, a fenced -/// block), which is what licenses the aliases. Do not use it on arbitrary -/// model output — see the module docs. -#[must_use] -pub fn parse_tool_call_value(value: &serde_json::Value) -> Option { - // Default to the permissive (tagged) behaviour: callers that reach a - // value through an explicit tool-call marker (`tool_calls` array, - // `` tags, ```tool_call blocks) accept the arg-key aliases. - parse_tool_call_value_aliased(value, true) -} - -/// Parse a single JSON value as a tool call. -/// -/// `allow_arg_aliases` controls whether the generic argument-key aliases in -/// [`TOOL_ARG_KEYS`] (notably the very generic `input`) are honoured for a -/// **bare** `{ "name": .., .. }` object. The whole-response fallback path -/// (`parse_tool_calls` on a top-level JSON object) passes `false`: there, a -/// normal model reply such as `{"name":"Alice","input":"hi"}` must not have -/// its `input` slurped into tool arguments and routed to execution -/// (bug-report-2026-05-26 A3 follow-up). The `function`-wrapped shape stays -/// permissive regardless — the `function` key is an unambiguous tool-call -/// marker. -fn parse_tool_call_value_aliased( - value: &serde_json::Value, - allow_arg_aliases: bool, -) -> Option { - if let Some(function) = value.get("function") { - let name = function - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim() - .to_string(); - if !name.is_empty() { - let arguments = first_args_by_keys(function); - return Some(ParsedToolCall { - name, - arguments, - id: None, - }); - } - } - - let name = value - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim() - .to_string(); - - if name.is_empty() { - return None; - } - - let arguments = if allow_arg_aliases { - first_args_by_keys(value) - } else { - // Whole-response bare-object fallback: require the canonical - // `arguments` key as an explicit tool-call marker. A plain JSON reply - // that merely carries a `name` (e.g. {"name":"Alice","input":…}) must - // stay plain text, not be dispatched as a tool call just because its - // name happens to match a registered tool (CodeRabbit, #2683). Tagged - // contexts (``/``, `tool_calls` array, `function` - // wrapper) reach this fn with `allow_arg_aliases = true` and keep the - // permissive behaviour. - parse_arguments_value(Some(value.get("arguments")?)) - }; - Some(ParsedToolCall { - name, - arguments, - id: None, - }) -} - -/// Parse a tagged JSON value containing one or more tool calls. -#[must_use] -pub fn parse_tool_calls_from_json_value(value: &serde_json::Value) -> Vec { - // Tagged contexts (callers reach here via an explicit tool-call marker) - // accept the argument-key aliases. - parse_tool_calls_from_json_value_aliased(value, true) -} - -/// Like [`parse_tool_calls_from_json_value`], but lets the caller forbid -/// generic arg-key aliases on a **bare** singleton/array object. The -/// `tool_calls`-keyed envelope always stays permissive — that key is an -/// unambiguous tool-call marker even on the whole-response path. -pub(crate) fn parse_tool_calls_from_json_value_aliased( - value: &serde_json::Value, - allow_arg_aliases: bool, -) -> Vec { - let mut calls = Vec::new(); - - if let Some(tool_calls) = value.get("tool_calls").and_then(|v| v.as_array()) { - for call in tool_calls { - // `tool_calls` entries are explicitly tool-call shaped → widen. - if let Some(parsed) = parse_tool_call_value_aliased(call, true) { - calls.push(parsed); - } - } - - if !calls.is_empty() { - return calls; - } - } - - if let Some(array) = value.as_array() { - for item in array { - if let Some(parsed) = parse_tool_call_value_aliased(item, allow_arg_aliases) { - calls.push(parsed); - } - } - return calls; - } - - if let Some(parsed) = parse_tool_call_value_aliased(value, allow_arg_aliases) { - calls.push(parsed); - } - - calls -} - -const TOOL_CALL_OPEN_TAGS: [&str; 4] = ["", "", "", ""]; - -pub(crate) fn find_first_tag<'a>(haystack: &str, tags: &'a [&'a str]) -> Option<(usize, &'a str)> { - tags.iter() - .filter_map(|tag| haystack.find(tag).map(|idx| (idx, *tag))) - .min_by_key(|(idx, _)| *idx) -} - -pub(crate) fn matching_tool_call_close_tag(open_tag: &str) -> Option<&'static str> { - match open_tag { - "" => Some(""), - "" => Some(""), - "" => Some(""), - "" => Some(""), - _ => None, - } -} - -/// A tool_call-family tag — ``, ``, `` — in ANY -/// open/close variant, tolerating sentinel-token pipes, a slash, and whitespace -/// leaked into the markers (`<|tool_call>`, ``, `<|tool_call|>`, -/// ``, …). Deliberately excludes `` (plural JSON key) -/// and `` (its own attribute parser). Used only to *locate* tags; -/// open/close is decided by pairing, not by this pattern. -static TOOL_CALL_TAG_RE: LazyLock> = - LazyLock::new(|| Regex::new(r"(?i)<[|/\s]*tool[_-]?call[|/\s]*>").ok()); - -/// Repair `` markers a weak model garbled by leaking its native -/// `<|…|>` sentinel-token pipes into the tags — e.g. `<|tool_call>call:{…}` -/// instead of `{…}`. Such shapes match no grammar, so the -/// whole call is dropped as narrative text and the tool never runs (observed -/// with a small Composio-toolkit sub-agent). -/// -/// Format-agnostic by construction: find every tool_call-family tag (any -/// pipe/slash garble) via [`TOOL_CALL_TAG_RE`] and pair them positionally — -/// 1st = open, 2nd = close, 3rd = open, … — rewriting each pair to canonical -/// `BODY` (and stripping a leading `call:` the model -/// sometimes emits). This sidesteps the unreliable per-tag open/close guess a -/// string-replace map needs. A trailing unpaired tag is left verbatim. Cheap -/// `Borrowed` no-op unless a *piped* tag is actually present, so well-formed -/// output — which the base parser already handles — and P-Format pipe args are -/// untouched. -fn normalize_garbled_tool_call_tags(s: &str) -> Cow<'_, str> { - // Garbling always leaks a `|` into a tag; no `|` anywhere → nothing to do. - if !s.contains('|') { - return Cow::Borrowed(s); - } - let Some(tool_call_tag_re) = TOOL_CALL_TAG_RE.as_ref() else { - return Cow::Borrowed(s); - }; - let tags: Vec<(usize, usize)> = tool_call_tag_re - .find_iter(s) - .map(|m| (m.start(), m.end())) - .collect(); - // Need at least one open/close pair, and at least one tag must actually be - // garbled (contain a pipe) — otherwise the base parser handles it verbatim, - // and P-Format `name[a|b]` args (pipes in the BODY, not the tags) are left - // alone. - if tags.len() < 2 || !tags.iter().any(|&(a, b)| s[a..b].contains('|')) { - return Cow::Borrowed(s); - } - let mut out = String::with_capacity(s.len()); - let mut cursor = 0usize; - // `as_chunks::<2>()` rather than `chunks_exact(2)`: the chunk size is a - // constant, so this hands back fixed-size arrays and the two destructurings - // below need no bounds check. `chunks_exact_to_as_chunks` (clippy, Rust - // 1.98) flags the older form. - for &[open, close] in tags.as_chunks::<2>().0 { - let (open_start, open_end) = open; - let (close_start, close_end) = close; - out.push_str(&s[cursor..open_start]); // text before the open tag, verbatim - out.push_str(""); - // Strip the `call:` prefix, then try to recover a Kimi-family - // `NAME{…}` argument-sentinel body into canonical JSON (#5119). When - // the body is already canonical JSON / P-Format the recovery is a no-op - // and the stripped body flows through unchanged. - let stripped = strip_call_prefix(&s[open_end..close_start]); - if let Some(recovered) = recover_sentinel_tool_call_body(stripped) { - // Recovered a Kimi-family `NAME{…}` sentinel body into canonical - // JSON. body_chars only (never the body itself — it may carry - // tool arguments with user data); stable `[agent_parse]` prefix - // so it aggregates with the other harness log families. - crate::telemetry::debug!( - body_chars = recovered.chars().count(), - "[agent_parse] recovered Kimi-family sentinel tool-call body into canonical JSON (#5119)" - ); - out.push_str(&recovered); - } else { - // A body still carrying the `<|"|>` arg-quote sentinel that - // recovery could NOT normalize is a new Kimi garble variant. - // Surface it (body_chars only — never the body: it may carry - // user data) so operators debugging a future unrecovered variant - // get a signal instead of a silently dropped tool call. - if stripped.contains(ARG_QUOTE_SENTINEL) { - crate::telemetry::warn!( - body_chars = stripped.chars().count(), - "[agent_parse] unrecovered Kimi-family sentinel tool-call body; passing through as text (#5119)" - ); - } - out.push_str(stripped); - } - out.push_str(""); - cursor = close_end; - } - // Trailing text, plus any final unpaired tag, verbatim. - out.push_str(&s[cursor..]); - Cow::Owned(out) -} - -/// Strip a leading `call:` some models emit right after the open tag, plus -/// surrounding whitespace, so the JSON / P-Format body underneath parses. -fn strip_call_prefix(body: &str) -> &str { - let trimmed = body.trim(); - trimmed - .strip_prefix("call:") - .map_or(trimmed, str::trim_start) -} - -/// The Kimi-K2-family argument-quote sentinel that leaks in place of a real `"` -/// around string values (`[<|"|>INBOX<|"|>]` instead of `["INBOX"]`). It is the -/// body-level sibling of the tag garble [`normalize_garbled_tool_call_tags`] -/// already repairs; see [`recover_sentinel_tool_call_body`]. -const ARG_QUOTE_SENTINEL: &str = "<|\"|>"; - -/// Recover a Kimi-K2-family garbled tool-call **body** into canonical -/// `{"name":…,"arguments":…}` JSON (#5119). -/// -/// The managed `burst`/`chat` tiers are Kimi-K2-family models; in text mode they -/// sometimes render a call as `NAME{…}` — the action name before a JSON-ish -/// argument object with **unquoted keys** and the `<|"|>` sentinel in place of -/// string quotes — e.g. `GMAIL_FETCH_EMAILS{label_ids:[<|"|>INBOX<|"|>],max_results:1}`. -/// After [`normalize_garbled_tool_call_tags`] fixes the surrounding tags this -/// body still matches neither the JSON nor the P-Format grammar, so the call is -/// dropped as narrative text and the tool never runs (the turn then loops). -/// -/// Recovery: replace the `<|"|>` sentinels with real quotes, split the leading -/// action name off the `{…}` object, quote the object's bare keys, and re-emit -/// as `{"name":"NAME","arguments":{…}}` for the existing JSON parser. Returns -/// `None` — leaving the body untouched — whenever the shape does not match: a -/// canonical JSON body (`{…}`, empty name), a P-Format body (`NAME[…]`, no `{`), -/// or the already-handled `call:{"name":…}` form all fall through unchanged. -fn recover_sentinel_tool_call_body(body: &str) -> Option { - let repaired = body.replace(ARG_QUOTE_SENTINEL, "\""); - let trimmed = repaired.trim(); - - // Shape must be `NAME{…}`: a bare action identifier immediately followed by - // a brace object. A body already starting with `{` yields an empty name and - // is left to the JSON parser; a `NAME[…]` P-Format body has no `{`. - let brace = trimmed.find('{')?; - let name = trimmed[..brace].trim(); - if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { - return None; - } - let object = trimmed[brace..].trim_end(); - if !object.starts_with('{') || !object.ends_with('}') { - return None; - } - - // Kimi renders object keys unquoted (`{label_ids: …}`); quote them so the - // result is strict JSON, then confirm it actually parses as an object before - // committing to the rewrite. - let quoted = quote_bare_json_object_keys(object); - let arguments: serde_json::Value = serde_json::from_str("ed).ok()?; - if !arguments.is_object() { - return None; - } - - serde_json::to_string(&serde_json::json!({ "name": name, "arguments": arguments })).ok() -} - -/// Quote every **bare** object key (`{label_ids: …}` → `{"label_ids": …}`) in a -/// JSON-ish string, tracking string context so a `:` or identifier inside a -/// value never triggers a spurious rewrite. Bare literal values -/// (`true`/`false`/`null`/numbers) are left untouched — they are valid JSON — -/// and already-quoted keys pass through unchanged. -fn quote_bare_json_object_keys(s: &str) -> String { - let mut out = String::with_capacity(s.len() + 16); - let mut in_string = false; - let mut escaped = false; - // Innermost container: `true` = object, `false` = array. A bare token may - // only be a key inside an object. - let mut in_object: Vec = Vec::new(); - // True right after a structural `{` or `,` — the only positions where a bare - // key may begin. - let mut expect_key = false; - let mut chars = s.chars().peekable(); - while let Some(c) = chars.next() { - if in_string { - out.push(c); - if escaped { - escaped = false; - } else if c == '\\' { - escaped = true; - } else if c == '"' { - in_string = false; - } - continue; - } - match c { - '"' => { - in_string = true; - expect_key = false; - out.push(c); - } - '{' | ',' => { - if c == '{' { - in_object.push(true); - } - expect_key = *in_object.last().unwrap_or(&false); - out.push(c); - } - '[' => { - in_object.push(false); - expect_key = false; - out.push(c); - } - '}' | ']' => { - in_object.pop(); - expect_key = false; - out.push(c); - } - c if c.is_whitespace() => out.push(c), // keep looking for a key - c if expect_key && (c.is_ascii_alphabetic() || c == '_') => { - out.push('"'); - out.push(c); - while let Some(&nc) = chars.peek() { - if nc.is_ascii_alphanumeric() || nc == '_' { - out.push(nc); - chars.next(); - } else { - break; - } - } - out.push('"'); - expect_key = false; - } - _ => { - expect_key = false; - out.push(c); - } - } - } - out -} - -/// ``) and Claude-native -/// attribute (``) forms. -const INVOKE_PREFIX: &str = "` open tag -/// (issue #3493). Matches `` form (next char `>`) is -/// intentionally skipped here; it is recognised as a literal tag with a JSON -/// body via [`TOOL_CALL_OPEN_TAGS`], preserving back-compat. -fn find_invoke_attr_tag(haystack: &str) -> Option { - let mut from = 0; - while let Some(rel) = haystack[from..].find(INVOKE_PREFIX) { - let idx = from + rel; - let after = &haystack[idx + INVOKE_PREFIX.len()..]; - match after.chars().next() { - Some(c) if c.is_whitespace() => return Some(idx), - _ => from = idx + INVOKE_PREFIX.len(), - } - } - None -} - -/// Scalar policy for `` values: a value that parses as JSON -/// (number, bool, null, array, object) is kept as that JSON type; anything -/// else — the common case of bare text — stays a string. Mirrors the tolerant -/// arg handling in [`parse_arguments_value`]. -fn parameter_scalar_value(raw: &str) -> serde_json::Value { - let trimmed = raw.trim(); - match serde_json::from_str::(trimmed) { - Ok( - value @ (serde_json::Value::Number(_) - | serde_json::Value::Bool(_) - | serde_json::Value::Null - | serde_json::Value::Array(_) - | serde_json::Value::Object(_)), - ) => value, - _ => serde_json::Value::String(trimmed.to_string()), - } -} - -/// Parse a Claude-native attribute-form invoke block whose text begins -/// immediately after the ``. `None` when the `name` attribute or the closing tag is -/// missing — the caller then leaves the markup as text rather than dropping it. -fn parse_invoke_attribute_block(after_prefix: &str) -> Option<(ParsedToolCall, usize)> { - static INVOKE_NAME_RE: LazyLock> = - LazyLock::new(|| Regex::new(r#"name\s*=\s*"([^"]*)""#).ok()); - static PARAMETER_RE: LazyLock> = LazyLock::new(|| { - Regex::new(r#"(?s)(.*?)"#).ok() - }); - - let invoke_name_re = INVOKE_NAME_RE.as_ref()?; - let parameter_re = PARAMETER_RE.as_ref()?; - - let open_end = after_prefix.find('>')?; - let attrs = &after_prefix[..open_end]; - let name = invoke_name_re - .captures(attrs) - .and_then(|c| c.get(1)) - .map(|m| m.as_str().trim().to_string()) - .filter(|n| !n.is_empty())?; - - let body = &after_prefix[open_end + 1..]; - let close_rel = body.find("")?; - let inner = &body[..close_rel]; - - let mut arguments = serde_json::Map::new(); - for cap in parameter_re.captures_iter(inner) { - // Groups 1 (name) and 2 (value) are mandatory in the pattern, so a - // captured match always has both — index access is safe. - let key = cap[1].trim(); - if key.is_empty() { - continue; - } - arguments.insert(key.to_string(), parameter_scalar_value(&cap[2])); - } - - let consumed = open_end + 1 + close_rel + "".len(); - Some(( - ParsedToolCall { - name, - arguments: serde_json::Value::Object(arguments), - id: None, - }, - consumed, - )) -} - -pub(crate) fn extract_first_json_value_with_end(input: &str) -> Option<(serde_json::Value, usize)> { - let trimmed = input.trim_start(); - let trim_offset = input.len().saturating_sub(trimmed.len()); - - for (byte_idx, ch) in trimmed.char_indices() { - if ch != '{' && ch != '[' { - continue; - } - - let slice = &trimmed[byte_idx..]; - let mut stream = serde_json::Deserializer::from_str(slice).into_iter::(); - if let Some(Ok(value)) = stream.next() { - let consumed = stream.byte_offset(); - if consumed > 0 { - return Some((value, trim_offset + byte_idx + consumed)); - } - } - } - - None -} - -pub(crate) fn strip_leading_close_tags(mut input: &str) -> &str { - loop { - let trimmed = input.trim_start(); - if !trimmed.starts_with("') else { - return ""; - }; - input = &trimmed[close_end + 1..]; - } -} - -/// Extract JSON values from a string. -/// -/// # Security Warning -/// -/// This function extracts ANY JSON objects/arrays from the input. It MUST only -/// be used on content that is already trusted to be from the LLM, such as -/// content inside `` tags where the LLM has explicitly indicated intent -/// to make a tool call. Do NOT use this on raw user input or content that -/// could contain prompt injection payloads. -#[must_use] -pub fn extract_json_values(input: &str) -> Vec { - let mut values = Vec::new(); - let trimmed = input.trim(); - if trimmed.is_empty() { - return values; - } - - if let Ok(value) = serde_json::from_str::(trimmed) { - values.push(value); - return values; - } - - let char_positions: Vec<(usize, char)> = trimmed.char_indices().collect(); - let mut idx = 0; - while idx < char_positions.len() { - let (byte_idx, ch) = char_positions[idx]; - if ch == '{' || ch == '[' { - let slice = &trimmed[byte_idx..]; - let mut stream = - serde_json::Deserializer::from_str(slice).into_iter::(); - if let Some(Ok(value)) = stream.next() { - let consumed = stream.byte_offset(); - if consumed > 0 { - values.push(value); - let next_byte = byte_idx + consumed; - while idx < char_positions.len() && char_positions[idx].0 < next_byte { - idx += 1; - } - continue; - } - } - } - idx += 1; - } - - values -} - -/// Find the end position of a JSON object by tracking balanced braces. -pub(crate) fn find_json_end(input: &str) -> Option { - let trimmed = input.trim_start(); - let offset = input.len() - trimmed.len(); - - if !trimmed.starts_with('{') { - return None; - } - - let mut depth = 0; - let mut in_string = false; - let mut escape_next = false; - - for (i, ch) in trimmed.char_indices() { - if escape_next { - escape_next = false; - continue; - } - - match ch { - '\\' if in_string => escape_next = true, - '"' => in_string = !in_string, - '{' if !in_string => depth += 1, - '}' if !in_string => { - depth -= 1; - if depth == 0 { - return Some(offset + i + ch.len_utf8()); - } - } - _ => {} - } - } - - None -} - -/// Parse GLM-style tool calls from response text. -/// GLM uses proprietary formats like: -/// - `browser_open/url>https://example.com` -/// - `shell/command>ls -la` -/// - `http_request/url>https://api.example.com` -pub(crate) fn map_glm_tool_alias(tool_name: &str) -> &str { - match tool_name { - "browser_open" | "browser" | "web_search" | "shell" | "bash" => "shell", - "http_request" | "http" => "http_request", - _ => tool_name, - } -} - -pub(crate) fn build_curl_command(url: &str) -> Option { - if !(url.starts_with("http://") || url.starts_with("https://")) { - return None; - } - - if url.chars().any(char::is_whitespace) { - return None; - } - - let escaped = url.replace('\'', "'\\''"); - Some(format!("curl -s '{escaped}'")) -} - -/// Parse GLM-style `tool/name>payload` calls from model text. -#[must_use] -pub fn parse_glm_style_tool_calls(text: &str) -> Vec<(String, serde_json::Value, Option)> { - let mut calls = Vec::new(); - - for line in text.lines() { - let line = line.trim(); - if line.is_empty() { - continue; - } - - // Format: tool_name/param>value or tool_name/{json} - if let Some(pos) = line.find('/') { - let tool_part = &line[..pos]; - let rest = &line[pos + 1..]; - - if tool_part.chars().all(|c| c.is_alphanumeric() || c == '_') { - let tool_name = map_glm_tool_alias(tool_part); - - if let Some(gt_pos) = rest.find('>') { - let param_name = rest[..gt_pos].trim(); - let value = rest[gt_pos + 1..].trim(); - - let arguments = match tool_name { - "shell" => { - if param_name == "url" { - let Some(command) = build_curl_command(value) else { - continue; - }; - serde_json::json!({"command": command}) - } else if value.starts_with("http://") || value.starts_with("https://") - { - if let Some(command) = build_curl_command(value) { - serde_json::json!({"command": command}) - } else { - serde_json::json!({"command": value}) - } - } else { - serde_json::json!({"command": value}) - } - } - "http_request" => { - serde_json::json!({"url": value, "method": "GET"}) - } - _ => serde_json::json!({param_name: value}), - }; - - calls.push((tool_name.to_string(), arguments, Some(line.to_string()))); - continue; - } - - if rest.starts_with('{') - && let Ok(json_args) = serde_json::from_str::(rest) - { - calls.push((tool_name.to_string(), json_args, Some(line.to_string()))); - } - } - } - } - - calls -} - -/// Parse tool calls from an LLM response that uses XML-style function calling. -/// -/// Expected format (common with system-prompt-guided tool use): -/// ```text -/// -/// {"name": "shell", "arguments": {"command": "ls"}} -/// -/// ``` -/// -/// Also accepts common tag variants (``, ``) for model -/// compatibility. -/// -/// Also supports JSON with `tool_calls` array from OpenAI-format responses. -#[must_use] -#[allow(clippy::too_many_lines)] -pub fn parse_tool_calls(response: &str) -> (String, Vec) { - let normalized = normalize_garbled_tool_call_tags(response); - let response = normalized.as_ref(); - let mut text_parts = Vec::new(); - let mut calls = Vec::new(); - let mut remaining = response; - - // First, try to parse as OpenAI-style JSON response with tool_calls array - // This handles providers like Minimax that return tool_calls in native JSON format - if let Ok(json_value) = serde_json::from_str::(response.trim()) { - // Whole-response parse: a bare top-level object/array is NOT an - // explicit tool-call marker, so forbid the generic arg-key aliases - // here (a plain `{"name":..,"input":..}` answer must stay text). - // The `tool_calls`-keyed envelope is still honoured (it carries its - // own marker) — handled inside the `_aliased` helper. - calls = parse_tool_calls_from_json_value_aliased(&json_value, false); - if !calls.is_empty() { - // If we found tool_calls, extract any content field as text - if let Some(content) = json_value.get("content").and_then(|v| v.as_str()) - && !content.trim().is_empty() - { - text_parts.push(content.trim().to_string()); - } - return (text_parts.join("\n"), calls); - } - } - - // Fall back to XML-style tool-call tag parsing. - loop { - let literal = find_first_tag(remaining, &TOOL_CALL_OPEN_TAGS); - let invoke_attr = find_invoke_attr_tag(remaining); - - // Choose the earliest-positioned recognised open tag. The bare - // `` literal and the attribute form `` never collide - // at one offset (one is followed by `>`, the other by whitespace), so a - // simple index comparison disambiguates them (issue #3493). - let use_invoke_attr = match (invoke_attr, literal.as_ref()) { - (Some(i), Some((l, _))) => i < *l, - (Some(_), None) => true, - _ => false, - }; - - if let Some(start) = invoke_attr.filter(|_| use_invoke_attr) { - let before = &remaining[..start]; - if !before.trim().is_empty() { - text_parts.push(before.trim().to_string()); - } - - let after_prefix = &remaining[start + INVOKE_PREFIX.len()..]; - if let Some((parsed, consumed)) = parse_invoke_attribute_block(after_prefix) { - calls.push(parsed); - remaining = &after_prefix[consumed..]; - continue; - } - - // Unparseable attribute-form block (no `name`/no close tag): leave - // it and the rest as text instead of silently dropping content. - crate::telemetry::warn!( - body_chars = after_prefix.chars().count(), - "[agent_parse] malformed attribute block: missing name or close tag" - ); - remaining = &remaining[start..]; - break; - } - - let Some((start, open_tag)) = literal else { - break; - }; - - // Everything before the tag is text. - let before = &remaining[..start]; - if !before.trim().is_empty() { - text_parts.push(before.trim().to_string()); - } - - let Some(close_tag) = matching_tool_call_close_tag(open_tag) else { - break; - }; - - let after_open = &remaining[start + open_tag.len()..]; - if let Some(close_idx) = after_open.find(close_tag) { - let inner = &after_open[..close_idx]; - let mut parsed_any = false; - let json_values = extract_json_values(inner); - for value in json_values { - let parsed_calls = parse_tool_calls_from_json_value(&value); - if !parsed_calls.is_empty() { - parsed_any = true; - calls.extend(parsed_calls); - } - } - - if !parsed_any { - // body_chars only (never the body itself — it may carry tool - // arguments with user data). Stable `[agent_parse]` prefix so - // it aggregates with the other harness log families. Surfaces - // how often the model emits an unparseable tool-call tag - // (bug-report-2026-05-26 A3). - crate::telemetry::warn!( - body_chars = inner.chars().count(), - "[agent_parse] malformed JSON: expected tool-call object in tag body" - ); - } - - remaining = &after_open[close_idx + close_tag.len()..]; - } else { - if let Some(json_end) = find_json_end(after_open) - && let Ok(value) = - serde_json::from_str::(&after_open[..json_end]) - { - let parsed_calls = parse_tool_calls_from_json_value(&value); - if !parsed_calls.is_empty() { - calls.extend(parsed_calls); - remaining = strip_leading_close_tags(&after_open[json_end..]); - continue; - } - } - - if let Some((value, consumed_end)) = extract_first_json_value_with_end(after_open) { - let parsed_calls = parse_tool_calls_from_json_value(&value); - if !parsed_calls.is_empty() { - calls.extend(parsed_calls); - remaining = strip_leading_close_tags(&after_open[consumed_end..]); - continue; - } - } - - remaining = &remaining[start..]; - break; - } - } - - // If XML tags found nothing, try markdown code blocks with tool_call language. - // Models behind OpenRouter sometimes output ```tool_call ... ``` or hybrid - // ```tool_call ... instead of structured API calls or XML tags. - if calls.is_empty() { - static MD_TOOL_CALL_RE: LazyLock> = LazyLock::new(|| { - Regex::new( - r"(?s)```(?:tool[_-]?call|invoke)\s*\n(.*?)(?:```|||)", - ) - .ok() - }); - let mut md_text_parts: Vec = Vec::new(); - let mut last_end = 0; - - for cap in MD_TOOL_CALL_RE - .as_ref() - .into_iter() - .flat_map(|regex| regex.captures_iter(response)) - { - let Some(full_match) = cap.get(0) else { - continue; - }; - let before = &response[last_end..full_match.start()]; - if !before.trim().is_empty() { - md_text_parts.push(before.trim().to_string()); - } - let inner = &cap[1]; - let json_values = extract_json_values(inner); - for value in json_values { - let parsed_calls = parse_tool_calls_from_json_value(&value); - calls.extend(parsed_calls); - } - last_end = full_match.end(); - } - - if !calls.is_empty() { - let after = &response[last_end..]; - if !after.trim().is_empty() { - md_text_parts.push(after.trim().to_string()); - } - text_parts = md_text_parts; - remaining = ""; - } - } - - // GLM-style tool calls (browser_open/url>https://..., shell/command>ls, etc.) - if calls.is_empty() { - let glm_calls = parse_glm_style_tool_calls(remaining); - if !glm_calls.is_empty() { - let mut cleaned_text = remaining.to_string(); - for (name, args, raw) in &glm_calls { - calls.push(ParsedToolCall { - name: name.clone(), - arguments: args.clone(), - id: None, - }); - if let Some(r) = raw { - cleaned_text = cleaned_text.replace(r, ""); - } - } - if !cleaned_text.trim().is_empty() { - text_parts.push(cleaned_text.trim().to_string()); - } - remaining = ""; - } - } - - // SECURITY: We do NOT fall back to extracting arbitrary JSON from the response - // here. That would enable prompt injection attacks where malicious content - // (e.g., in emails, files, or web pages) could include JSON that mimics a - // tool call. Tool calls MUST be explicitly wrapped in either: - // 1. OpenAI-style JSON with a "tool_calls" array - // 2. OpenHuman tool-call tags (, , ) - // 3. Markdown code blocks with tool_call/toolcall/tool-call language - // 4. Explicit GLM line-based call formats (e.g. `shell/command>...`) - // This ensures only the LLM's intentional tool calls are executed. - - // Remaining text after last tool call - if !remaining.trim().is_empty() { - text_parts.push(remaining.trim().to_string()); - } - - (text_parts.join("\n"), calls) -} - -/// P-Format-aware wrapper over [`parse_tool_calls`] (issue #4465). -/// -/// The migrated tinyagents parse path -/// (the native tool-use path) kept the XML/JSON/markdown/GLM -/// grammars but dropped the legacy **P-Format** positional grammar -/// (`name[arg1|arg2]`) — even though `PFormat` is the -/// default `ToolCallFormat` -/// and ~10 builtin agent prompts still *teach* the `name[a|b]` form. A model -/// that followed its own instructions therefore emitted calls that -/// [`parse_tool_calls`] logged as "malformed `` JSON" and silently -/// dropped, so the turn continued as if no tool was called. -/// -/// This restores parity by walking the ``-family tags and, for each -/// tag body, **preferring** the registry-driven P-Format parse -/// ([`pformat::parse_call`](super::pformat::parse_call)) and -/// **falling back** to the JSON entry the canonical parser produced at the same -/// ordinal position — the exact per-tag selection the legacy -/// `PFormatToolDispatcher` performed. This makes it a strict superset of -/// [`parse_tool_calls`]: -/// -/// - An **empty** `registry` (native/JSON agents advertise no positional -/// layout, or no tools at all) short-circuits to [`parse_tool_calls`], so -/// nothing changes for non-PFormat callers. -/// - A tag body that is not a valid `name[...]` positional call (e.g. a JSON -/// `{"name":..}` body, or an unregistered tool name) leaves -/// [`pformat::parse_call`](super::pformat::parse_call) -/// returning `None`, so the canonical JSON entry is used unchanged. -#[must_use] -pub fn parse_tool_calls_with_pformat( - response: &str, - registry: &super::pformat::PFormatRegistry, -) -> (String, Vec) { - let normalized = normalize_garbled_tool_call_tags(response); - let response = normalized.as_ref(); - // Canonical parse first: narrative text + JSON/XML/markdown/GLM calls. - let (narrative, json_calls) = parse_tool_calls(response); - - // Without a registry there is no positional layout to reconstruct — keep - // the canonical result verbatim (behaviour-neutral for non-PFormat paths). - if registry.is_empty() { - return (narrative, json_calls); - } - - // Walk the tags ourselves, preferring a P-Format body per tag and falling - // back to the JSON logic so calls retain their source order. - let mut combined: Vec = Vec::new(); - let mut remaining = response; - - while !remaining.is_empty() { - let Some((open_idx, open_tag)) = find_first_tag(remaining, &TOOL_CALL_OPEN_TAGS) else { - break; - }; - let Some(close_tag) = matching_tool_call_close_tag(open_tag) else { - break; - }; - let after_open = &remaining[open_idx + open_tag.len()..]; - let Some(close_idx) = after_open.find(close_tag) else { - break; - }; - let body = &after_open[..close_idx]; - - if let Some((name, arguments)) = super::pformat::parse_call(body, registry) { - // Do NOT log the arguments — a p-format body carries tool arguments - // that may contain user data (bug-report-2026-05-26 A3 parity). - crate::telemetry::debug!( - tool = name.as_str(), - "[agent_parse] recovered P-Format tool call (name[arg|arg]) the JSON pass dropped" - ); - combined.push(ParsedToolCall { - name, - arguments, - id: None, - }); - } else { - // Re-parse this tag body with the canonical JSON logic so a body - // holding several calls contributes all of them. - // - // Deliberately the *permissive* (alias-honouring) path rather than - // `parse_tool_calls`: a `` tag is an explicit tool-call - // marker, so the `args`/`parameters`/`input` aliases apply here. - let mut from_body: Vec = Vec::new(); - for value in extract_json_values(body) { - from_body.extend(parse_tool_calls_from_json_value(&value)); - } - // A tag body need not be JSON at all. GLM emits its own grammar - // (`shell/command>ls -la`), and once ANY tag in the response yields - // a p-format call this walk never falls back to the canonical - // result — so without this the GLM call is dropped and nothing - // reports it. Tried only when the JSON path found nothing, so a - // well-formed JSON body can never be double-counted. - if from_body.is_empty() { - from_body.extend(parse_glm_style_tool_calls(body).into_iter().map( - |(name, arguments, _raw)| ParsedToolCall { - name, - arguments, - id: None, - }, - )); - } - combined.extend(from_body); - } - - remaining = &after_open[close_idx + close_tag.len()..]; - } - - if combined.is_empty() { - // No `` tag recovered a positional call — the canonical - // result already covers JSON/XML/markdown/GLM grammars. - return (narrative, json_calls); - } - - // The literal-tag pass cannot visit provider-specific structures such as - // Claude's attribute-form ``. Preserve canonical calls - // it did not reconstruct, so mixed responses never lose an invocation. - for canonical in json_calls { - if !combined.iter().any(|call| { - call.name == canonical.name - && call.arguments == canonical.arguments - && call.id == canonical.id - }) { - combined.push(canonical); - } - } - - crate::telemetry::debug!( - parsed_tool_calls = combined.len(), - "[agent_parse] P-Format-aware parse produced combined tool-call set" - ); - (narrative, combined) -} - -#[cfg(test)] -#[path = "parse_test.rs"] -mod tests; diff --git a/crates/tinytools-agent/src/parse/call_object.rs b/crates/tinytools-agent/src/parse/call_object.rs new file mode 100644 index 0000000..f45aa25 --- /dev/null +++ b/crates/tinytools-agent/src/parse/call_object.rs @@ -0,0 +1,132 @@ +//! Reading a tool call out of an already-parsed JSON value. +//! +//! The shapes accepted, all seen from real models: +//! +//! * `{"name": "x", "arguments": {…}}` — canonical; +//! * `{"function": {"name": "x", "arguments": "{…}"}}` — the `OpenAI` wire +//! entry, with stringified arguments; +//! * `{"tool_calls": [ … ]}` — a whole wire message (Minimax); +//! * `[ {…}, {…} ]` — a bare array of calls; +//! * the argument-key aliases in [`crate::repair::args::ARGUMENT_KEYS`]. +//! +//! # The one rule that keeps this safe +//! +//! **Argument keys are aliased; tool names are not, and aliases need a +//! marker.** A model drifting from `arguments` to `args` still yields a usable +//! call. But a *bare* object — one the caller reached without a `` +//! tag, a `tool_calls` array, or a `function` wrapper — only counts as a call +//! when it carries the canonical `arguments` key or names a tool the caller +//! offered. Otherwise `{"name":"Alice","input":"hi"}`, an ordinary JSON +//! answer, would be dispatched as a phantom invocation. + +use serde_json::Value; + +use crate::repair::args; +use crate::types::{CallSource, ParsedToolCall}; + +/// Whether alias keys may be honoured on a bare `{"name": …}` object. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AliasPolicy { + /// Reached through an explicit marker: honour every alias. + Marked, + /// A bare object: require `arguments`, unless `name` is a known tool. + Bare, +} + +/// Reads one call object. +pub(crate) fn read_call( + value: &Value, + policy: AliasPolicy, + is_known: &dyn Fn(&str) -> bool, + source: CallSource, +) -> Option { + if let Some(function) = value.get("function") { + let name = function + .get("name") + .and_then(Value::as_str) + .unwrap_or("") + .trim(); + if !name.is_empty() { + return Some(ParsedToolCall::new( + name, + args::from_call_object(function), + source, + )); + } + } + + let name = value + .get("name") + .and_then(Value::as_str) + .unwrap_or("") + .trim(); + if name.is_empty() { + return None; + } + + let arguments = match policy { + AliasPolicy::Marked => args::from_call_object(value), + AliasPolicy::Bare if is_known(name) => args::from_call_object(value), + AliasPolicy::Bare => args::decode(Some(value.get("arguments")?)), + }; + Some(ParsedToolCall::new(name, arguments, source)) +} + +/// Reads every call in `value`: a `tool_calls` envelope, an array, or a +/// single object. The envelope is itself a marker, so its entries are always +/// read with [`AliasPolicy::Marked`]. +pub(crate) fn read_calls( + value: &Value, + policy: AliasPolicy, + is_known: &dyn Fn(&str) -> bool, + source: CallSource, +) -> Vec { + let mut calls = Vec::new(); + + if let Some(tool_calls) = value.get("tool_calls").and_then(Value::as_array) { + for call in tool_calls { + if let Some(parsed) = read_call(call, AliasPolicy::Marked, is_known, source) { + calls.push(parsed); + } + } + if !calls.is_empty() { + return calls; + } + } + + if let Some(array) = value.as_array() { + for item in array { + if let Some(parsed) = read_call(item, policy, is_known, source) { + calls.push(parsed); + } + } + return calls; + } + + if let Some(parsed) = read_call(value, policy, is_known, source) { + calls.push(parsed); + } + calls +} + +/// Reads one call object reached through an explicit marker. +#[must_use] +pub fn parse_tool_call_value(value: &Value) -> Option { + read_call( + value, + AliasPolicy::Marked, + &|_| false, + CallSource::TaggedJson, + ) +} + +/// Reads every call in a value reached through an explicit marker. +#[must_use] +pub fn parse_tool_calls_from_json_value(value: &Value) -> Vec { + read_calls( + value, + AliasPolicy::Marked, + &|_| false, + CallSource::TaggedJson, + ) +} diff --git a/crates/tinytools-agent/src/parse/grammar/bare_json.rs b/crates/tinytools-agent/src/parse/grammar/bare_json.rs new file mode 100644 index 0000000..4185b1c --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/bare_json.rs @@ -0,0 +1,55 @@ +//! A response that *is* a JSON value. +//! +//! Two shapes, both observed: +//! +//! * a provider that returns the whole wire message as text — +//! `{"content":"…","tool_calls":[…]}` (Minimax behind some gateways); +//! * a small model under `tool_choice: "required"` that writes the call +//! object as its entire reply, no markup, often with damaged quoting +//! (`llama3.2:3b` via Ollama: `{"name":"get_weather","parameters':{'city':"Paris"}}`). +//! +//! This is the one grammar with no marker, so it is the most tightly gated: +//! the **entire** trimmed response (after one surrounding fence) must be a +//! single JSON object or array, and a bare `{"name": …}` object only counts +//! when it carries the canonical `arguments` key or names a tool the caller +//! offered. Prose that quotes JSON has text outside the value and is left +//! alone; `{"name":"Alice","input":"hi"}` is left alone. + +use crate::parse::call_object::{AliasPolicy, read_calls}; +use crate::repair::json::{recover_object, strip_code_fence}; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The calls in a whole-response JSON value, plus any `content` text it +/// carried. `None` when the response is not one JSON value. +pub(crate) fn parse( + text: &str, + options: &ParseOptions<'_>, +) -> Option<(String, Vec)> { + let candidate = strip_code_fence(text.trim()); + let (first, last) = (candidate.chars().next()?, candidate.chars().last()?); + if !matches!((first, last), ('{', '}') | ('[', ']')) { + return None; + } + let is_known = |name: &str| options.knows(name); + + let value = match serde_json::from_str::(candidate) { + Ok(value) => value, + // A non-object that parsed strictly is not a call; do not "repair" + // it into one. Only an object-shaped candidate is worth recovering. + Err(_) if first == '{' => recover_object(candidate)?, + Err(_) => return None, + }; + + let calls = read_calls(&value, AliasPolicy::Bare, &is_known, CallSource::BareJson); + if calls.is_empty() { + return None; + } + let content = value + .get("content") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|c| !c.is_empty()) + .unwrap_or("") + .to_string(); + Some((content, calls)) +} diff --git a/crates/tinytools-agent/src/parse/grammar/glm.rs b/crates/tinytools-agent/src/parse/grammar/glm.rs new file mode 100644 index 0000000..3a02b2d --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/glm.rs @@ -0,0 +1,126 @@ +//! GLM's line-oriented `tool/param>value` calls. +//! +//! GLM models prompted for tools sometimes answer with one call per line: +//! +//! ```text +//! browser_open/url>https://example.com +//! shell/command>ls -la +//! http_request/url>https://api.example.com +//! custom/{"answer":42} +//! ``` +//! +//! This is not a marker-delimited grammar, so it never runs on arbitrary text +//! in the scan: it is tried on a tag body that decoded to nothing else, and +//! on the whole response only when no other grammar found a call. A handful +//! of GLM's own tool names are mapped onto the host's (`browser_open` → +//! `shell` with a `curl`), which is the one place this crate knows a tool +//! name. + +use serde_json::Value; + +use crate::types::{CallSource, ParsedToolCall}; + +/// Maps GLM's built-in tool names onto the host's. +#[must_use] +pub fn map_glm_tool_alias(tool_name: &str) -> &str { + match tool_name { + "browser_open" | "browser" | "web_search" | "shell" | "bash" => "shell", + "http_request" | "http" => "http_request", + _ => tool_name, + } +} + +/// A `curl` command fetching `url`, or `None` when it is not a plain +/// `http(s)` URL without whitespace. +#[must_use] +pub fn build_curl_command(url: &str) -> Option { + if !(url.starts_with("http://") || url.starts_with("https://")) { + return None; + } + if url.chars().any(char::is_whitespace) { + return None; + } + let escaped = url.replace('\'', "'\\''"); + Some(format!("curl -s '{escaped}'")) +} + +/// Every GLM-style call in `text`, as `(name, arguments, raw line)`. +#[must_use] +pub fn parse_glm_style_tool_calls(text: &str) -> Vec<(String, Value, Option)> { + let mut calls = Vec::new(); + + for line in text.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let Some(pos) = line.find('/') else { + continue; + }; + let tool_part = &line[..pos]; + let rest = &line[pos + 1..]; + if tool_part.is_empty() || !tool_part.chars().all(|c| c.is_alphanumeric() || c == '_') { + continue; + } + let tool_name = map_glm_tool_alias(tool_part); + + if let Some(gt_pos) = rest.find('>') { + let param_name = rest[..gt_pos].trim(); + let value = rest[gt_pos + 1..].trim(); + let arguments = match tool_name { + "shell" => { + if param_name == "url" { + let Some(command) = build_curl_command(value) else { + continue; + }; + serde_json::json!({ "command": command }) + } else if value.starts_with("http://") || value.starts_with("https://") { + match build_curl_command(value) { + Some(command) => serde_json::json!({ "command": command }), + None => serde_json::json!({ "command": value }), + } + } else { + serde_json::json!({ "command": value }) + } + } + "http_request" => serde_json::json!({ "url": value, "method": "GET" }), + _ => serde_json::json!({ param_name: value }), + }; + calls.push((tool_name.to_string(), arguments, Some(line.to_string()))); + continue; + } + + if rest.starts_with('{') + && let Ok(json_args) = serde_json::from_str::(rest) + { + calls.push((tool_name.to_string(), json_args, Some(line.to_string()))); + } + } + + calls +} + +/// [`parse_glm_style_tool_calls`] as [`ParsedToolCall`]s. +pub(crate) fn parse_lines(text: &str) -> Vec { + parse_glm_style_tool_calls(text) + .into_iter() + .map(|(name, arguments, _)| ParsedToolCall::new(name, arguments, CallSource::Glm)) + .collect() +} + +/// The calls in `text` plus the text with their lines removed. +pub(crate) fn parse_and_strip(text: &str) -> (String, Vec) { + let parsed = parse_glm_style_tool_calls(text); + if parsed.is_empty() { + return (text.to_string(), Vec::new()); + } + let mut cleaned = text.to_string(); + let mut calls = Vec::with_capacity(parsed.len()); + for (name, arguments, raw) in parsed { + calls.push(ParsedToolCall::new(name, arguments, CallSource::Glm)); + if let Some(raw) = raw { + cleaned = cleaned.replace(&raw, ""); + } + } + (cleaned, calls) +} diff --git a/crates/tinytools-agent/src/parse/grammar/harmony.rs b/crates/tinytools-agent/src/parse/grammar/harmony.rs new file mode 100644 index 0000000..2f27b28 --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/harmony.rs @@ -0,0 +1,104 @@ +//! `OpenAI` Harmony (gpt-oss) tool calls rendered as text. +//! +//! A gpt-oss model served without Harmony decoding writes its channel tokens +//! straight into the content: +//! +//! ```text +//! <|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{"city":"Paris"}<|call|> +//! ``` +//! +//! The call is the `to=` target of a `commentary` (or `analysis`) channel +//! message; the arguments are the `<|message|>` payload up to `<|call|>` +//! (or `<|end|>`). Any leading `<|start|>assistant` is furniture. A channel +//! message with no `to=` is ordinary reasoning or text, not a call, and is +//! left alone. + +use super::{Block, Decoded, Grammar, Probe, ScanMode, find_ci}; +use crate::repair::json::recover_object; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The Harmony grammar. +#[derive(Debug)] +pub(crate) struct Harmony; + +const CHANNEL: &str = "<|channel|>"; +const MESSAGE: &str = "<|message|>"; +const TERMINATORS: &[&str] = &["<|call|>", "<|end|>", "<|return|>"]; + +impl Grammar for Harmony { + fn source(&self) -> CallSource { + CallSource::Harmony + } + + fn probe(&self, text: &str, from: usize, _options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let mut cursor = from; + while let Some(idx) = find_ci(text, CHANNEL, cursor) { + let header_start = idx + CHANNEL.len(); + let Some(message_rel) = find_ci(text, MESSAGE, header_start) else { + if mode == ScanMode::Stream { + return Probe::Pending { start: idx }; + } + return Probe::None; + }; + let header = &text[header_start..message_rel]; + let Some(name) = target_name(header) else { + // A channel message that is not a call: skip past it. + cursor = message_rel + MESSAGE.len(); + continue; + }; + let payload_start = message_rel + MESSAGE.len(); + let after = &text[payload_start..]; + let terminator = TERMINATORS + .iter() + .filter_map(|t| after.find(t).map(|i| (i, i + t.len()))) + .min_by_key(|(i, _)| *i); + let Some((payload_end, term_end)) = terminator else { + if mode == ScanMode::Stream { + return Probe::Pending { start: idx }; + } + // Batch: the payload runs to the end of the text. + return found(idx, text.len(), &name, after); + }; + return found(idx, payload_start + term_end, &name, &after[..payload_end]); + } + Probe::None + } + + fn openers(&self) -> &'static [&'static str] { + &["<|channel|>", "<|start|>"] + } +} + +fn found(start: usize, end: usize, name: &str, payload: &str) -> Probe { + let arguments = recover_object(payload).unwrap_or_else(|| serde_json::json!({})); + Probe::Found(Block { + start, + end, + decoded: Decoded::Calls(vec![ParsedToolCall::new( + name, + arguments, + CallSource::Harmony, + )]), + }) +} + +/// The `to=` target of a channel header, with the `functions.` namespace +/// removed. `None` when the header carries no target. +fn target_name(header: &str) -> Option { + let idx = header.find("to=")?; + let rest = &header[idx + 3..]; + let raw: String = rest + .chars() + .take_while(|c| !c.is_whitespace() && *c != '<') + .collect(); + let name = raw + .strip_prefix("functions.") + .or_else(|| raw.strip_prefix("tools.")) + .unwrap_or(&raw) + .trim(); + if name.is_empty() { + None + } else { + Some(name.to_string()) + } +} diff --git a/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs new file mode 100644 index 0000000..499fa23 --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/invoke_xml.rs @@ -0,0 +1,251 @@ +//! `…` and its +//! relatives. +//! +//! One parser covers every XML-shaped call because they differ only in the +//! tag prefix and the attribute spelling: +//! +//! * Claude's native form, `…`; +//! * `DeepSeek` DSML, `<｜DSML｜invoke name="read">…` inside a +//! `<｜DSML｜tool_calls>` wrapper — with single or doubled bars, fullwidth +//! or ASCII, and an optional space after the marker (`<｜｜DSML｜｜ invoke`); +//! * namespaced variants such as ``; +//! * `` (Gemma) and `` (Llama / Qwen), with +//! `v` children. +//! +//! Arguments come from the parameter children when there are any, with the +//! DSML `{json}` envelope unwrapped; +//! otherwise from a JSON body, tolerating an orphan `` the model +//! left behind. Wrapper tags (``, ``, ``) +//! are protocol furniture and are removed without producing a call. + +use std::sync::LazyLock; + +use regex::Regex; + +use super::{Block, Decoded, Grammar, Probe, ScanMode, pending_opener, prefer_pending}; +use crate::repair::json::recover_object; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The invoke-XML grammar. +#[derive(Debug)] +pub(crate) struct InvokeXml; + +/// Optional tag prefix: a DSML marker or an XML namespace. +const PREFIX: &str = r"(?:[|｜]{1,2}\s*DSML\s*[|｜]{1,2}\s*|[A-Za-z_][\w.-]*:)?"; + +/// ``, ``, ``. +static OPEN_RE: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + r#"(?is)<{PREFIX}(?:invoke|function)(?:\s+[^>]*?\bname\s*=\s*"([^"]*)"[^>]*|\s*=\s*([^\s>,]+)[^>]*)>"# + )) + .ok() +}); + +/// Wrapper tags around a group of invokes, open or close. +static WRAPPER_RE: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + r"(?is)" + )) + .ok() +}); + +/// A closing tag ending an invoke: its own, ``, or a stray +/// `` some templates substitute. +static CLOSE_RE: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + r"(?is)" + )) + .ok() +}); + +/// `v` and `v`. +static PARAMETER_RE: LazyLock> = LazyLock::new(|| { + Regex::new(&format!( + r#"(?is)<{PREFIX}parameter(?:\s+[^>]*?\bname\s*=\s*"([^"]*)"[^>]*|\s*=\s*([^\s>]+)[^>]*)>(.*?)"# + )) + .ok() +}); + +/// An orphan closing parameter tag left in a JSON body. +static ORPHAN_PARAMETER_CLOSE_RE: LazyLock> = + LazyLock::new(|| Regex::new(&format!(r"(?is)")).ok()); + +impl Grammar for InvokeXml { + fn source(&self) -> CallSource { + CallSource::InvokeXml + } + + fn probe(&self, text: &str, from: usize, _options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let pending = pending_opener( + text, + from, + &["", + mode, + ); + prefer_pending(Self::probe_decided(text, from, mode), pending) + } + + fn openers(&self) -> &'static [&'static str] { + &[ + "", + "", + "", + "", + ] + } +} + +impl InvokeXml { + /// The next block whose opener is fully present. + fn probe_decided(text: &str, from: usize, mode: ScanMode) -> Probe { + let (Some(open_re), Some(wrapper_re), Some(close_re)) = + (OPEN_RE.as_ref(), WRAPPER_RE.as_ref(), CLOSE_RE.as_ref()) + else { + return Probe::None; + }; + let hay = &text[from..]; + + let wrapper = wrapper_re.find(hay); + let open = open_re.captures(hay); + + // A wrapper tag before the next invoke is furniture: remove it alone. + // A bare `` opener with no invoke anywhere after it is a + // prose mention (the JSON key, say) and stays. + if let Some(w) = wrapper + && open + .as_ref() + .is_none_or(|o| w.start() < o.get(0).map_or(usize::MAX, |m| m.start())) + && (open.is_some() || is_closer_or_prefixed(w.as_str())) + { + return Probe::Found(Block { + start: from + w.start(), + end: from + w.end(), + decoded: Decoded::Noise, + }); + } + + let Some(open) = open else { + return Probe::None; + }; + let Some(open_match) = open.get(0) else { + return Probe::None; + }; + let name = open + .get(1) + .or_else(|| open.get(2)) + .map_or("", |m| m.as_str().trim()); + let start = from + open_match.start(); + let body_start = from + open_match.end(); + let after = &text[body_start..]; + + let close = close_re.find(after).map(|m| (m.start(), m.end())); + let Some((body_end, close_end)) = close else { + if mode == ScanMode::Stream { + return Probe::Pending { start }; + } + return Probe::Found(Block { + start, + end: text.len(), + decoded: Decoded::Verbatim, + }); + }; + + let body = &after[..body_end]; + let end = body_start + close_end; + if name.is_empty() { + return Probe::Found(Block { + start, + end, + decoded: Decoded::Malformed { + body_chars: body.chars().count(), + }, + }); + } + + let arguments = decode_arguments(body); + Probe::Found(Block { + start, + end, + decoded: Decoded::Calls(vec![ParsedToolCall::new( + name, + arguments, + CallSource::InvokeXml, + )]), + }) + } +} + +/// Whether a wrapper tag is a closer or carries a DSML / namespace prefix — +/// either is unambiguous protocol furniture even with no invoke in sight. +fn is_closer_or_prefixed(tag: &str) -> bool { + let inner = &tag[1..]; + inner.starts_with('/') + || !inner.starts_with(|c: char| c.is_ascii_alphabetic()) + || inner.contains(':') +} + +/// Arguments from parameter children or a JSON body. +fn decode_arguments(body: &str) -> serde_json::Value { + let Some(parameter_re) = PARAMETER_RE.as_ref() else { + return serde_json::json!({}); + }; + + let mut parameters = serde_json::Map::new(); + for cap in parameter_re.captures_iter(body) { + let key = cap + .get(1) + .or_else(|| cap.get(2)) + .map_or("", |m| m.as_str().trim()); + if key.is_empty() { + continue; + } + let raw = cap.get(3).map_or("", |m| m.as_str()); + parameters.insert(key.to_string(), scalar_value(raw)); + } + + if !parameters.is_empty() { + // DSML's `{json}` envelope. + if parameters.len() == 1 + && let Some(envelope @ serde_json::Value::Object(_)) = parameters.get("arguments") + { + return envelope.clone(); + } + return serde_json::Value::Object(parameters); + } + + let stripped = ORPHAN_PARAMETER_CLOSE_RE.as_ref().map_or_else( + || body.to_string(), + |re| re.replace_all(body, "").into_owned(), + ); + let stripped = stripped.trim(); + if stripped.is_empty() { + return serde_json::json!({}); + } + if let Some(object) = recover_object(stripped) { + return object; + } + serde_json::json!({ "input": stripped }) +} + +/// A parameter value: JSON when it parses as a number, bool, null, array or +/// object; otherwise the trimmed text. +fn scalar_value(raw: &str) -> serde_json::Value { + let trimmed = raw.trim(); + match serde_json::from_str::(trimmed) { + Ok( + value @ (serde_json::Value::Number(_) + | serde_json::Value::Bool(_) + | serde_json::Value::Null + | serde_json::Value::Array(_) + | serde_json::Value::Object(_)), + ) => value, + _ => serde_json::Value::String(trimmed.to_string()), + } +} diff --git a/crates/tinytools-agent/src/parse/grammar/mistral.rs b/crates/tinytools-agent/src/parse/grammar/mistral.rs new file mode 100644 index 0000000..0e575ca --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/mistral.rs @@ -0,0 +1,117 @@ +//! Mistral `[TOOL_CALLS]` blocks rendered as text. +//! +//! Mistral's templates mark a call with a literal `[TOOL_CALLS]` token +//! followed by either a JSON array of `{"name":…,"arguments":…}` objects +//! (v3 templates) or `NAME[ARGS]{…}` (v11 and later). Served through a route +//! that does not decode the token, both arrive in the content verbatim. + +use super::{Block, Decoded, Grammar, Probe, ScanMode}; +use crate::parse::call_object::{AliasPolicy, read_calls}; +use crate::parse::json_values::extract_first_json_value_with_end; +use crate::repair::json::recover_object; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The Mistral grammar. +#[derive(Debug)] +pub(crate) struct Mistral; + +const MARKER: &str = "[TOOL_CALLS]"; +const ARGS: &str = "[ARGS]"; + +impl Grammar for Mistral { + fn source(&self) -> CallSource { + CallSource::Mistral + } + + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let Some(rel) = text[from..].find(MARKER) else { + return Probe::None; + }; + let start = from + rel; + let body_start = start + MARKER.len(); + let after = &text[body_start..]; + let is_known = |name: &str| options.knows(name); + + // v3: a JSON array (or single object) right after the marker. + if let Some((value, consumed)) = extract_first_json_value_with_end(after) + && after[..consumed].trim_start().starts_with(['[', '{']) + && !after.trim_start().starts_with(ARGS) + { + let calls = read_calls(&value, AliasPolicy::Marked, &is_known, CallSource::Mistral); + if !calls.is_empty() { + return Probe::Found(Block { + start, + end: body_start + consumed, + decoded: Decoded::Calls(calls), + }); + } + } + + // v11+: `NAME[ARGS]{…}`, possibly several in a row. + let mut calls = Vec::new(); + let mut cursor = 0usize; + loop { + let rest = &after[cursor..]; + let Some(args_rel) = rest.find(ARGS) else { + break; + }; + let name = rest[..args_rel].trim(); + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.') + { + break; + } + let payload = &rest[args_rel + ARGS.len()..]; + let Some((value, consumed)) = extract_first_json_value_with_end(payload) else { + break; + }; + let arguments = if value.is_object() { + value + } else { + recover_object(&payload[..consumed]).unwrap_or_else(|| serde_json::json!({})) + }; + calls.push(ParsedToolCall::new(name, arguments, CallSource::Mistral)); + cursor += args_rel + ARGS.len() + consumed; + } + if !calls.is_empty() { + // The v11 form allows a second call to follow directly with no + // fresh `[TOOL_CALLS]` marker (`NAME[ARGS]{…}NAME2[ARGS]{…}`). + // If the buffered text ends right where the trailing bytes + // could still grow into another such name, a stream fragment + // has not necessarily finished the block — finalizing now would + // drop the continuation call the moment it arrives split across + // a fragment boundary. Hold the whole block until either more + // text disambiguates it or the stream ends. + let could_continue = mode == ScanMode::Stream + && after[cursor..] + .trim_start() + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '.'); + if could_continue { + return Probe::Pending { start }; + } + return Probe::Found(Block { + start, + end: body_start + cursor, + decoded: Decoded::Calls(calls), + }); + } + + if mode == ScanMode::Stream { + return Probe::Pending { start }; + } + Probe::Found(Block { + start, + end: body_start, + decoded: Decoded::Malformed { + body_chars: after.chars().count(), + }, + }) + } + + fn openers(&self) -> &'static [&'static str] { + &["[TOOL_CALLS]"] + } +} diff --git a/crates/tinytools-agent/src/parse/grammar/mod.rs b/crates/tinytools-agent/src/parse/grammar/mod.rs new file mode 100644 index 0000000..3b996cb --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/mod.rs @@ -0,0 +1,166 @@ +//! The surface syntaxes a tool call can arrive in, one module each. +//! +//! A grammar answers one question: *starting at a byte offset, where is the +//! next block I recognise, and what calls does it hold?* It does not know +//! about protected ranges, about other grammars, or about how the narrative +//! text is assembled — that is the scan engine in [`crate::parse`]. Keeping +//! grammars this narrow is what makes adding one a local change: a new file +//! here, a line in [`GRAMMARS`], and every caller — batch, streaming, every +//! dialect — sees it. +//! +//! Order in [`GRAMMARS`] only breaks ties between grammars whose openers sit +//! at the same byte; the engine otherwise takes the earliest opener. + +pub(crate) mod bare_json; +pub(crate) mod glm; +pub(crate) mod harmony; +pub(crate) mod invoke_xml; +pub(crate) mod mistral; +pub(crate) mod sentinel; +pub(crate) mod tagged; + +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// Whether the text may still grow. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ScanMode { + /// The response is complete: an opener with no closer is recovered where + /// the grammar can (a balanced JSON body) and otherwise kept as text. + Batch, + /// More fragments may arrive: an opener with no closer is reported as + /// pending so the caller holds it back. + Stream, +} + +/// What a recognised block decoded to. +#[derive(Debug)] +pub(crate) enum Decoded { + /// One or more calls; the block is removed from the narrative. + Calls(Vec), + /// A recognised block whose body is not a call; removed from the + /// narrative and reported. + Malformed { + /// Length of the body in characters. + body_chars: usize, + }, + /// Protocol furniture with no call of its own (a `` wrapper + /// tag, a stray closing tag); removed from the narrative silently. + Noise, + /// A recognised opener whose block cannot be decoded; kept in the + /// narrative verbatim. + Verbatim, +} + +/// A block a grammar recognised. +#[derive(Debug)] +pub(crate) struct Block { + /// Byte offset of the block's first byte. + pub(crate) start: usize, + /// Byte offset just past the block. + pub(crate) end: usize, + /// What it decoded to. + pub(crate) decoded: Decoded, +} + +/// The result of asking a grammar for its next block. +#[derive(Debug)] +pub(crate) enum Probe { + /// Nothing recognised at or after the offset. + None, + /// A complete block. + Found(Block), + /// An opener at `start` with no closer yet (streaming only). + Pending { + /// Byte offset of the opener. + start: usize, + }, +} + +/// One surface syntax. +pub(crate) trait Grammar: Sync { + /// Which [`CallSource`] this grammar produces. + fn source(&self) -> CallSource; + + /// The next block at or after `from`. + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe; + + /// Literal prefixes that open one of this grammar's blocks, used by the + /// stream scrubber to hold back a partially received opener. Compared + /// ASCII-case-insensitively. + fn openers(&self) -> &'static [&'static str]; +} + +/// Every scan grammar, in tie-break order. +pub(crate) static GRAMMARS: &[&dyn Grammar] = &[ + &invoke_xml::InvokeXml, + &sentinel::Sentinel, + &harmony::Harmony, + &mistral::Mistral, + &tagged::Tagged, +]; + +/// Every opener prefix across all scan grammars. +pub(crate) fn all_openers() -> impl Iterator { + GRAMMARS + .iter() + .flat_map(|grammar| grammar.openers().iter().copied()) +} + +/// Case-insensitive `find` for an ASCII needle. +pub(crate) fn find_ci(haystack: &str, needle: &str, from: usize) -> Option { + if needle.is_empty() || from > haystack.len() { + return None; + } + let hay = haystack.as_bytes(); + let nee = needle.as_bytes(); + if nee.len() > hay.len() { + return None; + } + (from..=hay.len() - nee.len()) + .filter(|&i| haystack.is_char_boundary(i)) + .find(|&i| hay[i..i + nee.len()].eq_ignore_ascii_case(nee)) +} + +/// In [`ScanMode::Stream`], the earliest occurrence at or after `from` of one +/// of `literals` that cannot be decided yet: it is not the prefix of a longer +/// word and no `terminator` follows it anywhere in the text, so its block +/// cannot possibly be complete. `None` in batch mode. +pub(crate) fn pending_opener( + text: &str, + from: usize, + literals: &[&str], + terminator: &str, + mode: ScanMode, +) -> Option { + if mode != ScanMode::Stream { + return None; + } + let mut earliest: Option = None; + for literal in literals { + let mut cursor = from; + while let Some(idx) = find_ci(text, literal, cursor) { + let after = idx + literal.len(); + let word_continues = text[after..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphanumeric()); + if !word_continues && !text[after..].contains(terminator) { + if earliest.is_none_or(|e| idx < e) { + earliest = Some(idx); + } + break; + } + cursor = after; + } + } + earliest +} + +/// Prefers a pending opener over a later (or absent) decided block. +pub(crate) fn prefer_pending(probe: Probe, pending: Option) -> Probe { + match (probe, pending) { + (Probe::Found(block), Some(start)) if start < block.start => Probe::Pending { start }, + (Probe::None, Some(start)) => Probe::Pending { start }, + (probe, _) => probe, + } +} diff --git a/crates/tinytools-agent/src/parse/grammar/sentinel.rs b/crates/tinytools-agent/src/parse/grammar/sentinel.rs new file mode 100644 index 0000000..9b3611e --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/sentinel.rs @@ -0,0 +1,221 @@ +//! Chat-template sentinel tokens emitted verbatim as text. +//! +//! When an OpenAI-compatible route serves a model through its own chat +//! template without decoding the template's special tokens, the call arrives +//! as the raw tokens. Two families are common enough to matter: +//! +//! * **`DeepSeek`** (R1, V3): +//! `<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>NAME\n```json\n{…}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>`, +//! or the shorter `<｜tool▁call▁begin｜>NAME<｜tool▁sep｜>{…}<｜tool▁call▁end｜>`; +//! * **Kimi K2**: +//! `<|tool_calls_section_begin|><|tool_call_begin|>functions.NAME:0<|tool_call_argument_begin|>{…}<|tool_call_end|><|tool_calls_section_end|>`. +//! +//! Both appear with fullwidth (`｜`) or ASCII (`|`) bars and with `▁` or `_` +//! between words, so the markers are matched by a small regex rather than a +//! literal table. The section wrappers are furniture and are removed; each +//! `call_begin … call_end` block yields one call. + +use std::sync::LazyLock; + +use regex::Regex; + +use super::{Block, Decoded, Grammar, Probe, ScanMode, pending_opener, prefer_pending}; +use crate::parse::call_object::{AliasPolicy, read_calls}; +use crate::repair::json::{recover_object, strip_code_fence}; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The sentinel-token grammar. +#[derive(Debug)] +pub(crate) struct Sentinel; + +/// `<|tool_call_begin|>` in every bar/underscore spelling. +static CALL_BEGIN_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<[|｜]tool[▁_]call[▁_]begin[|｜]>").ok()); + +/// `<|tool_call_end|>`. +static CALL_END_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<[|｜]tool[▁_]call[▁_]end[|｜]>").ok()); + +/// The section wrappers: `<|tool_calls_begin|>`, `<|tool_calls_end|>`, +/// `<|tool_calls_section_begin|>`, `<|tool_calls_section_end|>`. +static WRAPPER_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"<[|｜]tool[▁_]calls(?:[▁_]section)?[▁_](?:begin|end)[|｜]>").ok() +}); + +/// `DeepSeek`'s name/arguments separator. +static SEP_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<[|｜]tool[▁_]sep[|｜]>").ok()); + +/// Kimi's name/arguments separator. +static ARGUMENT_BEGIN_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"<[|｜]tool[▁_]call[▁_]argument[▁_]begin[|｜]>").ok()); + +impl Grammar for Sentinel { + fn source(&self) -> CallSource { + CallSource::Sentinel + } + + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let pending = pending_opener( + text, + from, + &[ + "<|tool_call", + "<｜tool▁call", + "<|tool_calls", + "<｜tool▁calls", + ], + ">", + mode, + ); + prefer_pending(Self::probe_decided(text, from, options, mode), pending) + } + + fn openers(&self) -> &'static [&'static str] { + &[ + "<|tool_call", + "<｜tool▁call", + "<|tool_calls", + "<｜tool▁calls", + ] + } +} + +impl Sentinel { + /// The next block whose opener is fully present. + fn probe_decided(text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let (Some(begin_re), Some(end_re), Some(wrapper_re)) = ( + CALL_BEGIN_RE.as_ref(), + CALL_END_RE.as_ref(), + WRAPPER_RE.as_ref(), + ) else { + return Probe::None; + }; + let hay = &text[from..]; + let wrapper = wrapper_re.find(hay); + let begin = begin_re.find(hay); + + if let Some(w) = wrapper + && begin.is_none_or(|b| w.start() < b.start()) + { + return Probe::Found(Block { + start: from + w.start(), + end: from + w.end(), + decoded: Decoded::Noise, + }); + } + let Some(begin) = begin else { + return Probe::None; + }; + + let start = from + begin.start(); + let body_start = from + begin.end(); + let after = &text[body_start..]; + let Some(end) = end_re.find(after) else { + if mode == ScanMode::Stream { + return Probe::Pending { start }; + } + return Probe::Found(Block { + start, + end: text.len(), + decoded: Decoded::Verbatim, + }); + }; + + let body = &after[..end.start()]; + let block_end = body_start + end.end(); + let calls = decode_body(body, options); + let decoded = if calls.is_empty() { + Decoded::Malformed { + body_chars: body.chars().count(), + } + } else { + Decoded::Calls(calls) + }; + Probe::Found(Block { + start, + end: block_end, + decoded, + }) + } +} + +/// The body between `call_begin` and `call_end`, in every known layout. +fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec { + let is_known = |name: &str| options.knows(name); + + // Kimi: `functions.NAME:0<|tool_call_argument_begin|>{…}`. + if let Some(re) = ARGUMENT_BEGIN_RE.as_ref() + && let Some(sep) = re.find(body) + { + let name = kimi_name(&body[..sep.start()]); + let arguments = arguments_from(&body[sep.end()..]); + if !name.is_empty() { + return vec![ParsedToolCall::new(name, arguments, CallSource::Sentinel)]; + } + return Vec::new(); + } + + // DeepSeek: `function<|tool_sep|>NAME\n```json\n{…}\n```` or `NAME<|tool_sep|>{…}`. + if let Some(re) = SEP_RE.as_ref() + && let Some(sep) = re.find(body) + { + let left = body[..sep.start()].trim(); + let right = body[sep.end()..].trim(); + let (name, raw_arguments) = if matches!(left, "function" | "tool" | "functions") { + match right.split_once('\n') { + Some((first, rest)) => (first.trim(), rest), + None => (right, ""), + } + } else { + (left, right) + }; + if name.is_empty() { + return Vec::new(); + } + return vec![ParsedToolCall::new( + name, + arguments_from(raw_arguments), + CallSource::Sentinel, + )]; + } + + // No separator: a `{"name":…,"arguments":…}` object, possibly fenced. + let unfenced = strip_code_fence(body); + let value = serde_json::from_str::(unfenced) + .ok() + .or_else(|| recover_object(unfenced)); + match value { + Some(value) => read_calls(&value, AliasPolicy::Marked, &is_known, CallSource::Sentinel), + None => Vec::new(), + } +} + +/// `functions.NAME:0` → `NAME`. +fn kimi_name(raw: &str) -> &str { + let trimmed = raw.trim(); + let without_index = trimmed.rsplit_once(':').map_or(trimmed, |(head, tail)| { + if tail.chars().all(|c| c.is_ascii_digit()) { + head + } else { + trimmed + } + }); + without_index + .strip_prefix("functions.") + .unwrap_or(without_index) + .trim() +} + +/// Arguments from a possibly fenced JSON object; empty object when absent. +fn arguments_from(raw: &str) -> serde_json::Value { + let unfenced = strip_code_fence(raw.trim()); + if unfenced.is_empty() { + return serde_json::json!({}); + } + serde_json::from_str::(unfenced) + .ok() + .filter(serde_json::Value::is_object) + .or_else(|| recover_object(unfenced)) + .unwrap_or_else(|| serde_json::json!({})) +} diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs new file mode 100644 index 0000000..ec32fb3 --- /dev/null +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -0,0 +1,353 @@ +//! `…` and everything models do to it. +//! +//! The baseline text protocol — JSON (or P-Format) inside a tag — arrives in +//! more spellings than any other grammar because every chat template has its +//! own, and gateways garble the tag markers themselves: +//! +//! * spelling variants ``, ``, and the bare ``; +//! * an attribute form `` (Hermes / `DeepSeek` +//! templates); +//! * sentinel pipes leaked into the markers, in any position: +//! `<|tool_call>…`, `<|tool_call|>…<|tool_call|>`, +//! `…`; +//! * a `call:` prefix before the body; +//! * a fenced block instead of a tag, ```` ```tool_call … ``` ````, sometimes +//! closed by a stray ``; +//! * a body that is a Kimi `NAME{…}` object with unquoted keys and `<|"|>` +//! quote sentinels; +//! * a body wrapped in its own ```` ```json ```` fence. +//! +//! Tags are paired **positionally**: the first tag-family marker after the +//! cursor opens a block, the next one closes it. That is what makes the +//! garbled forms parse without a per-variant open/close table, and it is +//! safe because these tags never nest. + +use std::sync::LazyLock; + +use regex::Regex; + +use super::{Block, Decoded, Grammar, Probe, ScanMode, find_ci, pending_opener, prefer_pending}; +use crate::parse::call_object::{AliasPolicy, read_calls}; +use crate::parse::json_values::{ + extract_first_json_value_with_end, extract_json_values, find_json_end, strip_leading_close_tags, +}; +use crate::repair::json::{recover_object, strip_code_fence}; +use crate::types::{CallSource, ParseOptions, ParsedToolCall}; + +/// The tagged-JSON grammar. +#[derive(Debug)] +pub(crate) struct Tagged; + +/// Any tag-family marker: ``, ``, ``, with +/// pipes, a slash, or whitespace leaked in, and an optional attribute list. +/// `` (plural, a JSON key) and `` do not match: +/// the name must end at a pipe, slash, whitespace, or `>`. +static TAG_RE: LazyLock> = + LazyLock::new(|| Regex::new(r"(?i)<[|/\s]*tool[_-]?call(?:[|/\s]*|\s+[^>]*)>").ok()); + +/// Openers a fenced block can carry. +const FENCE_OPENERS: &[&str] = &["```tool_call", "```toolcall", "```tool-call", "```invoke"]; + +/// Kimi-family argument-quote sentinel that leaks in place of `"`. +const ARG_QUOTE_SENTINEL: &str = "<|\"|>"; + +/// A located opener. +struct Opener { + start: usize, + body_start: usize, + kind: OpenerKind, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum OpenerKind { + /// A tag-family marker; closed by the next tag-family marker. + Tag, + /// The bare `` literal; closed by ``. + Invoke, + /// A fenced block; closed by ```` ``` ```` or a stray closing tag. + Fence, +} + +impl Grammar for Tagged { + fn source(&self) -> CallSource { + CallSource::TaggedJson + } + + fn probe(&self, text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let pending = pending_opener( + text, + from, + &["", + mode, + ); + prefer_pending(Self::probe_decided(text, from, options, mode), pending) + } + + fn openers(&self) -> &'static [&'static str] { + &[ + "", + "```tool_call", + "```toolcall", + "```tool-call", + "```invoke", + ] + } +} + +impl Tagged { + /// The next block whose opener is fully present. + fn probe_decided(text: &str, from: usize, options: &ParseOptions<'_>, mode: ScanMode) -> Probe { + let Some(opener) = next_opener(text, from) else { + return Probe::None; + }; + let after = &text[opener.body_start..]; + + let close = match opener.kind { + OpenerKind::Tag => TAG_RE + .as_ref() + .and_then(|re| re.find(after)) + .map(|m| (m.start(), m.end())), + OpenerKind::Invoke => after.find("").map(|i| (i, i + "".len())), + OpenerKind::Fence => fence_close(after), + }; + + if let Some((body_end, close_end)) = close { + let body = &after[..body_end]; + let end = opener.body_start + close_end; + let calls = decode_body(body, options); + let decoded = if calls.is_empty() { + Decoded::Malformed { + body_chars: body.chars().count(), + } + } else { + Decoded::Calls(calls) + }; + return Probe::Found(Block { + start: opener.start, + end, + decoded, + }); + } + + if mode == ScanMode::Stream { + return Probe::Pending { + start: opener.start, + }; + } + + // Batch: no closer. Recover a balanced JSON body if one starts here. + let recovered = find_json_end(after) + .and_then(|json_end| { + serde_json::from_str::(&after[..json_end]) + .ok() + .map(|value| (value, json_end)) + }) + .or_else(|| extract_first_json_value_with_end(after)); + if let Some((value, consumed)) = recovered { + let calls = read_calls( + &value, + AliasPolicy::Marked, + &|name| options.knows(name), + CallSource::TaggedJson, + ); + if !calls.is_empty() { + let rest = &after[consumed..]; + let stripped = strip_leading_close_tags(rest); + let end = text.len() - stripped.len(); + return Probe::Found(Block { + start: opener.start, + end, + decoded: Decoded::Calls(calls), + }); + } + } + Probe::Found(Block { + start: opener.start, + end: text.len(), + decoded: Decoded::Verbatim, + }) + } +} + +/// The earliest opener at or after `from`: a non-closing tag-family marker, +/// the bare `` literal, or a fence opener. +fn next_opener(text: &str, from: usize) -> Option { + let mut best: Option = None; + let consider = |best: &mut Option, candidate: Opener| { + if best.as_ref().is_none_or(|b| candidate.start < b.start) { + *best = Some(candidate); + } + }; + + if let Some(re) = TAG_RE.as_ref() { + for m in re.find_iter(&text[from..]) { + // A marker with a slash is a closer, never an opener. + let inner = &m.as_str()[1..]; + if inner.trim_start_matches(['|', ' ', '\t']).starts_with('/') { + continue; + } + consider( + &mut best, + Opener { + start: from + m.start(), + body_start: from + m.end(), + kind: OpenerKind::Tag, + }, + ); + break; + } + } + + if let Some(idx) = find_ci(text, "", from) { + consider( + &mut best, + Opener { + start: idx, + body_start: idx + "".len(), + kind: OpenerKind::Invoke, + }, + ); + } + + for fence in FENCE_OPENERS { + let mut cursor = from; + while let Some(idx) = find_ci(text, fence, cursor) { + let after = &text[idx + fence.len()..]; + // The language must end here (`tool_call` not `tool_calls`), and + // the body starts on the next line. + let rest = after.trim_start_matches([' ', '\t']); + if let Some(nl) = rest + .strip_prefix('\n') + .or_else(|| rest.strip_prefix("\r\n")) + { + consider( + &mut best, + Opener { + start: idx, + body_start: text.len() - nl.len(), + kind: OpenerKind::Fence, + }, + ); + break; + } + cursor = idx + fence.len(); + } + } + + best +} + +/// The closer of a fenced block: a closing fence, a stray tag-family closer, +/// or ``, whichever comes first. +fn fence_close(after: &str) -> Option<(usize, usize)> { + let mut best: Option<(usize, usize)> = None; + let mut consider = |candidate: Option<(usize, usize)>| { + if let Some(c) = candidate + && best.is_none_or(|b| c.0 < b.0) + { + best = Some(c); + } + }; + consider(after.find("```").map(|i| (i, i + 3))); + consider( + TAG_RE + .as_ref() + .and_then(|re| re.find(after)) + .filter(|m| { + m.as_str()[1..] + .trim_start_matches(['|', ' ']) + .starts_with('/') + }) + .map(|m| (m.start(), m.end())), + ); + consider(after.find("").map(|i| (i, i + "".len()))); + best +} + +/// Everything a tag body can be, tried in order. +pub(crate) fn decode_body(body: &str, options: &ParseOptions<'_>) -> Vec { + let body = strip_call_prefix(body); + let is_known = |name: &str| options.knows(name); + + if let Some(registry) = options.registry + && let Some((name, arguments)) = crate::pformat::parse_call(body, registry) + { + return vec![ParsedToolCall::new(name, arguments, CallSource::PFormat)]; + } + + if let Some(recovered) = recover_sentinel_body(body) + && let Ok(value) = serde_json::from_str::(&recovered) + { + let calls = read_calls( + &value, + AliasPolicy::Marked, + &is_known, + CallSource::TaggedJson, + ); + if !calls.is_empty() { + return calls; + } + } + + let unfenced = strip_code_fence(body); + let mut calls = Vec::new(); + for value in extract_json_values(unfenced) { + calls.extend(read_calls( + &value, + AliasPolicy::Marked, + &is_known, + CallSource::TaggedJson, + )); + } + if !calls.is_empty() { + return calls; + } + + if let Some(value) = recover_object(unfenced) { + let calls = read_calls( + &value, + AliasPolicy::Marked, + &is_known, + CallSource::TaggedJson, + ); + if !calls.is_empty() { + return calls; + } + } + + super::glm::parse_lines(body) +} + +/// Strips a leading `call:` some models emit right after the open tag. +fn strip_call_prefix(body: &str) -> &str { + let trimmed = body.trim(); + trimmed + .strip_prefix("call:") + .map_or(trimmed, str::trim_start) +} + +/// Recovers a Kimi-K2-family `NAME{…}` body — the action name before a +/// JSON-ish object with unquoted keys and `<|"|>` in place of string quotes — +/// into canonical `{"name":…,"arguments":…}` JSON. `None` when the shape does +/// not match: a body already starting with `{`, a P-Format `NAME[…]` body, +/// or trailing text after the object all fall through unchanged. +pub(crate) fn recover_sentinel_body(body: &str) -> Option { + let repaired = body.replace(ARG_QUOTE_SENTINEL, "\""); + let trimmed = repaired.trim(); + let brace = trimmed.find('{')?; + let name = trimmed[..brace].trim(); + if name.is_empty() || !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return None; + } + let object = trimmed[brace..].trim_end(); + if !object.starts_with('{') || !object.ends_with('}') { + return None; + } + let arguments = recover_object(object)?; + serde_json::to_string(&serde_json::json!({ "name": name, "arguments": arguments })).ok() +} diff --git a/crates/tinytools-agent/src/parse/json_values.rs b/crates/tinytools-agent/src/parse/json_values.rs new file mode 100644 index 0000000..108ad67 --- /dev/null +++ b/crates/tinytools-agent/src/parse/json_values.rs @@ -0,0 +1,127 @@ +//! Locating JSON values inside free text. +//! +//! # Security +//! +//! These scanners pull *any* JSON value out of a string. They must only run on +//! text the model has explicitly marked as a tool call — the body of a +//! `` tag, an `` block, a fenced `tool_call` block. Running +//! them over arbitrary output would let a tool result that contains +//! `{"name":"shell",…}` (an email, a web page) be dispatched as a call. + +use serde_json::Value; + +/// Every JSON value in `input`, in order. A string that is one JSON value +/// yields exactly that value. +#[must_use] +pub fn extract_json_values(input: &str) -> Vec { + let mut values = Vec::new(); + let trimmed = input.trim(); + if trimmed.is_empty() { + return values; + } + + if let Ok(value) = serde_json::from_str::(trimmed) { + values.push(value); + return values; + } + + let char_positions: Vec<(usize, char)> = trimmed.char_indices().collect(); + let mut idx = 0; + while idx < char_positions.len() { + let (byte_idx, ch) = char_positions[idx]; + if ch == '{' || ch == '[' { + let slice = &trimmed[byte_idx..]; + let mut stream = serde_json::Deserializer::from_str(slice).into_iter::(); + if let Some(Ok(value)) = stream.next() { + let consumed = stream.byte_offset(); + if consumed > 0 { + values.push(value); + let next_byte = byte_idx + consumed; + while idx < char_positions.len() && char_positions[idx].0 < next_byte { + idx += 1; + } + continue; + } + } + } + idx += 1; + } + + values +} + +/// The first JSON value in `input` and the byte offset just past it. +#[must_use] +pub(crate) fn extract_first_json_value_with_end(input: &str) -> Option<(Value, usize)> { + let trimmed = input.trim_start(); + let trim_offset = input.len().saturating_sub(trimmed.len()); + + for (byte_idx, ch) in trimmed.char_indices() { + if ch != '{' && ch != '[' { + continue; + } + let slice = &trimmed[byte_idx..]; + let mut stream = serde_json::Deserializer::from_str(slice).into_iter::(); + if let Some(Ok(value)) = stream.next() { + let consumed = stream.byte_offset(); + if consumed > 0 { + return Some((value, trim_offset + byte_idx + consumed)); + } + } + } + + None +} + +/// The byte offset just past the object that opens at the start of `input` +/// (after leading whitespace), found by tracking balanced braces. +#[must_use] +pub(crate) fn find_json_end(input: &str) -> Option { + let trimmed = input.trim_start(); + let offset = input.len() - trimmed.len(); + + if !trimmed.starts_with('{') { + return None; + } + + let mut depth = 0; + let mut in_string = false; + let mut escape_next = false; + + for (i, ch) in trimmed.char_indices() { + if escape_next { + escape_next = false; + continue; + } + match ch { + '\\' if in_string => escape_next = true, + '"' => in_string = !in_string, + '{' if !in_string => depth += 1, + '}' if !in_string => { + depth -= 1; + if depth == 0 { + return Some(offset + i + ch.len_utf8()); + } + } + _ => {} + } + } + + None +} + +/// Drops any run of leading closing tags (``) and the whitespace around +/// them. A truncated closing tag with no `>` consumes the rest. +#[must_use] +pub(crate) fn strip_leading_close_tags(mut input: &str) -> &str { + loop { + let trimmed = input.trim_start(); + if !trimmed.starts_with("') else { + return ""; + }; + input = &trimmed[close_end + 1..]; + } +} diff --git a/crates/tinytools-agent/src/parse/mod.rs b/crates/tinytools-agent/src/parse/mod.rs new file mode 100644 index 0000000..ef5e2fc --- /dev/null +++ b/crates/tinytools-agent/src/parse/mod.rs @@ -0,0 +1,284 @@ +//! Turning model text back into tool calls. +//! +//! A model that supports native tool use hands back structured calls and none +//! of this is needed. Everything else — prompt-guided models, local models, +//! providers whose native mode is unavailable, and native models that narrate +//! a call as text anyway — emits tool calls as *text*, in whatever shape the +//! model was trained to produce. This module turns that text back into calls. +//! +//! # How a response is read +//! +//! 1. If the **whole** response is one JSON value, it is read as a call +//! envelope (the bare-JSON grammar) and nothing else runs. +//! 2. Otherwise the text is scanned left to right. At each step every +//! grammar reports its next block; the earliest one wins, its +//! calls are collected, and the scan resumes past it. Text between blocks +//! is the narrative. Blocks inside a [`protected`] code fence are skipped. +//! 3. If the scan found nothing, the GLM line grammar is tried on the +//! narrative. +//! 4. Every call's name is resolved against the offered tools +//! ([`crate::repair::name`]) when the caller supplied them. +//! +//! # Why it is this forgiving, and where it stops +//! +//! Each accommodation exists because a model actually produced it and the +//! alternative was dropping a well-formed call and burning an iteration. The +//! permissiveness is bounded on purpose: +//! +//! * **A call needs a marker.** Only the bare-JSON path runs without one, and +//! it requires the entire response to be the value. +//! * **Argument keys are aliased; tool names are not.** `args` / `parameters` +//! / `input` are honoured behind a marker. On a bare object they need the +//! canonical `arguments` key or a name the caller offered, so a plain JSON +//! answer is never a phantom call. +//! * **Names are repaired only to a unique offered tool.** Nothing is invented. +//! * **Code fences with a language are examples**, not calls. +//! * **P-Format refuses to invent argument names** for a tool it does not know. +//! +//! # What the host still owns +//! +//! Ids: this module never mints one. Execution: permission, sandboxing, +//! timeouts, and the unknown-tool policy are host decisions; a call whose +//! name did not resolve is still returned, flagged in the diagnostics. + +pub(crate) mod call_object; +pub(crate) mod grammar; +pub(crate) mod json_values; +pub mod protected; + +use std::ops::Range; + +use crate::repair; +use crate::types::{CallSource, ParseDiagnostic, ParseOptions, ParseOutcome, ParsedToolCall}; +use grammar::{Decoded, GRAMMARS, Probe, ScanMode}; + +pub use call_object::{parse_tool_call_value, parse_tool_calls_from_json_value}; +pub use grammar::glm::{build_curl_command, map_glm_tool_alias, parse_glm_style_tool_calls}; +pub use json_values::extract_json_values; + +/// Parses a complete model response. +#[must_use] +pub fn parse_text(text: &str, options: &ParseOptions<'_>) -> ParseOutcome { + if options.allow_bare_json + && let Some((content, calls)) = grammar::bare_json::parse(text, options) + { + return finalize(content, calls, Vec::new(), options); + } + + let scan = scan(text, options, ScanMode::Batch); + let parts: Vec<&str> = scan + .kept + .iter() + .map(|range| text[range.clone()].trim()) + .filter(|part| !part.is_empty()) + .collect(); + let mut calls = scan.calls; + let diagnostics = scan.diagnostics; + + if calls.is_empty() { + let joined = parts.join("\n"); + let (cleaned, glm_calls) = grammar::glm::parse_and_strip(&joined); + if !glm_calls.is_empty() { + calls = glm_calls; + return finalize(cleaned.trim().to_string(), calls, diagnostics, options); + } + } + finalize(parts.join("\n"), calls, diagnostics, options) +} + +/// Name resolution and the diagnostics it produces. +fn finalize( + text: String, + calls: Vec, + diagnostics: Vec, + options: &ParseOptions<'_>, +) -> ParseOutcome { + let (calls, diagnostics) = resolve_names(calls, diagnostics, options); + ParseOutcome { + text, + calls, + diagnostics, + } +} + +/// Resolves every text-recovered call's name against the offered tools. +pub(crate) fn resolve_names( + mut calls: Vec, + mut diagnostics: Vec, + options: &ParseOptions<'_>, +) -> (Vec, Vec) { + for call in &mut calls { + if call.source == CallSource::Native { + continue; + } + let resolution = repair::name::resolve(&call.name, options.known_tools); + if resolution.repaired { + crate::telemetry::debug!( + from_chars = call.name.chars().count(), + to = resolution.name.as_str(), + "[agent_parse] repaired tool name" + ); + diagnostics.push(ParseDiagnostic::NameRepaired { + from: std::mem::take(&mut call.name), + to: resolution.name.clone(), + }); + call.name = resolution.name; + } else if options.has_known_tools() && !resolution.known { + diagnostics.push(ParseDiagnostic::UnknownTool { + name: call.name.clone(), + }); + } + } + (calls, diagnostics) +} + +/// The result of one scan pass. +#[derive(Debug, Default)] +pub(crate) struct Scan { + /// Byte ranges of the text kept as narrative, in order. + pub(crate) kept: Vec>, + /// Calls in source order. + pub(crate) calls: Vec, + /// What was dropped or left unterminated. + pub(crate) diagnostics: Vec, + /// In [`ScanMode::Stream`], the offset of an opener whose block has not + /// closed yet. Text from there on must be held back. + pub(crate) pending: Option, +} + +/// One left-to-right pass over `text`. +pub(crate) fn scan(text: &str, options: &ParseOptions<'_>, mode: ScanMode) -> Scan { + let ranges = protected::fence_ranges(text); + let mut out = Scan::default(); + let mut from = 0usize; + + loop { + let mut best: Option = None; + let mut best_source = CallSource::TaggedJson; + let mut best_start = usize::MAX; + for grammar in GRAMMARS { + let mut cursor = from; + let candidate = loop { + match grammar.probe(text, cursor, options, mode) { + Probe::None => break None, + Probe::Found(block) => { + if let Some(end) = protected::protected_end(&ranges, block.start) { + if end <= cursor { + break None; + } + cursor = end; + continue; + } + break Some((block.start, Probe::Found(block))); + } + Probe::Pending { start } => { + if let Some(end) = protected::protected_end(&ranges, start) { + if end <= cursor { + break None; + } + cursor = end; + continue; + } + break Some((start, Probe::Pending { start })); + } + } + }; + if let Some((start, probe)) = candidate + && start < best_start + { + best_start = start; + best_source = grammar.source(); + best = Some(probe); + } + } + + match best { + None => { + // A fence still open at the end of the text has not yet + // received its closing fence (or its protected content is + // still arriving). Releasing it now, mid-stream, would drop + // the only record that the next fragment is still inside + // it — so hold it back like any other pending opener rather + // than draining it as narrative. + let open_fence = (mode == ScanMode::Stream) + .then(|| protected::open_fence_start(text)) + .flatten() + .filter(|&start| start >= from); + match open_fence { + Some(start) => { + out.kept.push(from..start); + out.pending = Some(start); + } + None => out.kept.push(from..text.len()), + } + break; + } + Some(Probe::Pending { start }) => { + out.kept.push(from..start); + out.pending = Some(start); + break; + } + Some(Probe::Found(block)) => { + let source = best_source; + match block.decoded { + Decoded::Calls(calls) => { + out.kept.push(from..block.start); + out.calls.extend(calls); + } + Decoded::Malformed { body_chars } => { + out.kept.push(from..block.start); + crate::telemetry::warn!( + body_chars, + "[agent_parse] malformed tool-call block: body did not decode to a call" + ); + out.diagnostics + .push(ParseDiagnostic::MalformedBlock { source, body_chars }); + } + Decoded::Noise => { + out.kept.push(from..block.start); + } + Decoded::Verbatim => { + out.kept.push(from..block.end); + crate::telemetry::warn!( + "[agent_parse] unterminated tool-call block kept as text" + ); + out.diagnostics + .push(ParseDiagnostic::UnterminatedBlock { source }); + } + } + from = block.end.max(block.start + 1).min(text.len()); + } + Some(Probe::None) => break, + } + } + out +} + +/// Parses a response with default options. +/// +/// Kept for callers that predate [`parse_text`]; equivalent to +/// `parse_text(response, &ParseOptions::new()).into_parts()`. +#[must_use] +pub fn parse_tool_calls(response: &str) -> (String, Vec) { + parse_text(response, &ParseOptions::new()).into_parts() +} + +/// Parses a response with a P-Format registry, preferring a positional body +/// inside each tag and falling back to JSON per tag. +#[must_use] +pub fn parse_tool_calls_with_pformat( + response: &str, + registry: &crate::PFormatRegistry, +) -> (String, Vec) { + let options = ParseOptions::new().with_registry(registry); + parse_text(response, &options).into_parts() +} + +/// Normalizes an argument value, decoding stringified JSON when possible. +#[must_use] +pub fn parse_arguments_value(raw: Option<&serde_json::Value>) -> serde_json::Value { + repair::args::decode(raw) +} + +#[cfg(test)] +mod test; diff --git a/crates/tinytools-agent/src/parse/protected.rs b/crates/tinytools-agent/src/parse/protected.rs new file mode 100644 index 0000000..6673b9b --- /dev/null +++ b/crates/tinytools-agent/src/parse/protected.rs @@ -0,0 +1,115 @@ +//! Spans of a response in which nothing is a tool call. +//! +//! A model explaining a tool protocol, quoting a transcript, or writing a +//! shell script that happens to contain `` puts that text inside a +//! fenced code block. Dispatching a call from there would execute an +//! *example*. So a fence with a language tag protects its contents from every +//! grammar in [`crate::parse`]. +//! +//! Two deliberate exceptions keep real calls parseable: +//! +//! * a fence whose language *is* a tool-call marker (```` ```tool_call ````) +//! is a call, not an example, and is handled by the tagged grammar; +//! * a fence with **no** language tag is not protected. Small models wrap a +//! genuine call in a bare fence far more often than they quote one, and a +//! quoted example almost always carries a language. +//! +//! An unclosed fence protects to the end of the text. + +use std::ops::Range; + +#[cfg(test)] +mod test; + +/// Info-string languages that mark a fence as a tool call rather than a code +/// example. +pub const TOOL_CALL_LANGUAGES: &[&str] = + &["tool_call", "toolcall", "tool-call", "invoke", "tool_calls"]; + +/// Byte ranges of protected fenced blocks, in order, non-overlapping. An +/// unclosed trailing fence is included, protecting to the end of `text`. +#[must_use] +pub fn fence_ranges(text: &str) -> Vec> { + scan_fences(text).0 +} + +/// The start of a fence that is still open at the end of `text`, if any. +/// +/// Used by the stream scrubber: a fence opener with no closing fence yet +/// must not be released as safe narrative text, because the fragments that +/// close it — and the protected content in between — have not arrived. A +/// fence that has already closed returns `None`, even though its content is +/// still reported by [`fence_ranges`]. +#[must_use] +pub fn open_fence_start(text: &str) -> Option { + scan_fences(text).1 +} + +/// One pass over `text` tracking fence state. Returns every closed fenced +/// range plus the start of a trailing fence still open when the text ends. +fn scan_fences(text: &str) -> (Vec>, Option) { + let mut ranges = Vec::new(); + let mut open: Option<(usize, char, usize)> = None; // (start, fence char, fence len) + let mut offset = 0; + for line in text.split_inclusive('\n') { + let line_start = offset; + offset += line.len(); + let stripped = line.trim_start_matches(' '); + if line.len() - stripped.len() > 3 { + continue; + } + let Some(fence_char) = stripped.chars().next().filter(|c| *c == '`' || *c == '~') else { + continue; + }; + let fence_len = stripped.chars().take_while(|c| *c == fence_char).count(); + if fence_len < 3 { + continue; + } + let info = stripped[fence_len..].trim(); + match open { + None => { + let language = info.split_whitespace().next().unwrap_or(""); + let is_tool_call = TOOL_CALL_LANGUAGES + .iter() + .any(|lang| lang.eq_ignore_ascii_case(language)); + if !language.is_empty() && !is_tool_call { + open = Some((line_start, fence_char, fence_len)); + } + } + Some((start, open_char, open_len)) => { + // CommonMark forbids an info string on a closing fence; here + // it is allowed, because a model closing a fenced argument + // block often puts the next protocol marker on the same + // line (```` ```<｜tool▁call▁end｜> ````). + if fence_char == open_char && fence_len >= open_len { + // The range ends at the closing backticks, not the line + // end, so a marker on the same line is scanned. + let indent = line.len() - stripped.len(); + let fence_bytes: usize = + stripped.chars().take(fence_len).map(char::len_utf8).sum(); + ranges.push(start..line_start + indent + fence_bytes); + open = None; + } + } + } + } + if let Some((start, _, _)) = open { + ranges.push(start..text.len()); + } + (ranges, open.map(|(start, _, _)| start)) +} + +/// Whether `position` falls inside any of `ranges`. +#[must_use] +pub fn is_protected(ranges: &[Range], position: usize) -> bool { + ranges.iter().any(|range| range.contains(&position)) +} + +/// The end of the protected range containing `position`, if any. +#[must_use] +pub fn protected_end(ranges: &[Range], position: usize) -> Option { + ranges + .iter() + .find(|range| range.contains(&position)) + .map(|range| range.end) +} diff --git a/crates/tinytools-agent/src/parse/protected/test.rs b/crates/tinytools-agent/src/parse/protected/test.rs new file mode 100644 index 0000000..e7ff97a --- /dev/null +++ b/crates/tinytools-agent/src/parse/protected/test.rs @@ -0,0 +1,49 @@ +//! Unit tests for protected fenced-block detection. +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +use super::{fence_ranges, is_protected, protected_end}; + +#[test] +fn a_fence_indented_more_than_three_spaces_is_not_a_fence() { + // CommonMark treats a fence indented four or more spaces as an + // indented code block, not a fence, so it must not open a protected + // range even though it carries a language tag. + let text = " ```rust\nfn main() {}\n ```\n"; + assert!(fence_ranges(text).is_empty()); +} + +#[test] +fn a_two_backtick_run_is_not_a_fence() { + // A fence needs at least three backticks (or tildes); shorter runs are + // inline code spans, not fence delimiters. + let text = "``json\n{\"a\":1}\n``\n"; + assert!(fence_ranges(text).is_empty()); +} + +#[test] +fn is_protected_reports_positions_inside_and_outside_a_fence() { + let text = "before\n```json\nx\n```\nafter"; + let ranges = fence_ranges(text); + assert_eq!(ranges.len(), 1); + + let fence_start = text.find("```json").expect("fence marker present"); + let inside = fence_start + 4; + assert!(is_protected(&ranges, inside)); + + let outside = text.find("before").expect("prefix present"); + assert!(!is_protected(&ranges, outside)); +} + +#[test] +fn protected_end_locates_the_close_of_the_containing_fence() { + let text = "```json\nx\n```\nafter"; + let ranges = fence_ranges(text); + let fence_start = 0; + let inside = text.find("").expect("marker present"); + + assert_eq!(protected_end(&ranges, inside), Some(ranges[0].end)); + assert_eq!(protected_end(&ranges, fence_start), Some(ranges[0].end)); + + let after = text.rfind("after").expect("suffix present"); + assert_eq!(protected_end(&ranges, after), None); +} diff --git a/crates/tinytools-agent/src/parse/test/bare_json.rs b/crates/tinytools-agent/src/parse/test/bare_json.rs new file mode 100644 index 0000000..a0f4d16 --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/bare_json.rs @@ -0,0 +1,102 @@ +//! A response that is entirely one JSON value. + +use super::{parse, parse_known}; +use crate::types::{CallSource, ParseOptions}; + +#[test] +fn a_wire_message_with_tool_calls_array_parses() { + let native = serde_json::json!({ + "content": "native text", + "tool_calls": [ { "name": "echo", "arguments": { "value": "one" } } ] + }) + .to_string(); + let (text, calls) = parse(&native); + assert_eq!(text, "native text"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].source, CallSource::BareJson); +} + +#[test] +fn a_bare_object_with_canonical_arguments_parses() { + let (text, calls) = parse(r#"{"name":"echo","arguments":{"value":"hi"}}"#); + assert!(text.is_empty()); + assert_eq!(calls.len(), 1); +} + +#[test] +fn a_bare_object_with_only_an_alias_is_plain_text() { + let (text, calls) = parse(r#"{"name":"Alice","input":{"value":"hi"}}"#); + assert!( + calls.is_empty(), + "a JSON answer must not become a phantom call" + ); + assert_eq!(text, r#"{"name":"Alice","input":{"value":"hi"}}"#); +} + +#[test] +fn a_bare_object_naming_a_known_tool_may_use_an_alias() { + let outcome = parse_known( + r#"{"name":"get_weather","parameters":{"city":"Paris"}}"#, + &["get_weather"], + ); + assert_eq!(outcome.calls.len(), 1); + assert_eq!(outcome.calls[0].arguments["city"], "Paris"); +} + +#[test] +fn llama_bare_object_with_mismatched_quotes_is_repaired() { + let outcome = parse_known( + r#"{"name":"get_weather","parameters':{'city':"Paris"}}"#, + &["get_weather"], + ); + assert_eq!(outcome.calls.len(), 1); + assert_eq!( + outcome.calls[0].arguments, + serde_json::json!({ "city": "Paris" }) + ); + assert!(outcome.text.is_empty()); +} + +#[test] +fn a_bare_object_inside_a_code_fence_parses() { + let (_, calls) = + parse("```json\n{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}\n```"); + assert_eq!(calls.len(), 1); +} + +#[test] +fn bare_recovery_never_swallows_a_genuine_text_answer() { + for text in [ + "The weather in Paris is mild today.", + r#"You could send {"name":"get_weather"} to that endpoint."#, + r#"{"city":"Paris","temperature":17}"#, + r#"{"name":42}"#, + r#""just a string""#, + "[1, 2, 3]", + ] { + let (cleaned, calls) = parse(text); + assert!( + calls.is_empty(), + "{text:?} must not be recovered as a tool call" + ); + assert_eq!(cleaned, text); + } +} + +#[test] +fn bare_json_can_be_disabled() { + let options = ParseOptions::new().without_bare_json(); + let outcome = crate::parse::parse_text(r#"{"name":"echo","arguments":{}}"#, &options); + assert!(outcome.calls.is_empty()); +} + +#[test] +fn a_bare_array_of_calls_parses() { + let array = serde_json::json!([ + { "name": "echo", "arguments": { "value": "two" } }, + { "name": " " } + ]) + .to_string(); + let (_, calls) = parse(&array); + assert_eq!(calls.len(), 1); +} diff --git a/crates/tinytools-agent/src/parse/test/engine.rs b/crates/tinytools-agent/src/parse/test/engine.rs new file mode 100644 index 0000000..d3e709d --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/engine.rs @@ -0,0 +1,217 @@ +//! The scan engine: protected fences, name resolution, helpers, diagnostics. + +use super::{parse, parse_known}; +use crate::parse::json_values::{ + extract_first_json_value_with_end, find_json_end, strip_leading_close_tags, +}; +use crate::parse::protected::fence_ranges; +use crate::parse::{ + extract_json_values, parse_arguments_value, parse_tool_call_value, + parse_tool_calls_from_json_value, +}; +use crate::types::ParseDiagnostic; + +#[test] +fn a_call_inside_a_language_fence_is_an_example_not_a_call() { + let text = "Here is the format:\n```bash\n{\"name\":\"shell\",\"arguments\":{\"command\":\"rm -rf /\"}}\n```\nDo not run it."; + let (cleaned, calls) = parse(text); + assert!(calls.is_empty(), "fenced example must not dispatch"); + assert_eq!(cleaned, text); +} + +#[test] +fn a_call_inside_a_bare_fence_still_parses() { + let text = + "```\n{\"name\":\"shell\",\"arguments\":{\"command\":\"ls\"}}\n```"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1); +} + +#[test] +fn a_call_after_a_closed_fence_parses() { + let text = "```python\nprint('')\n```\n{\"name\":\"echo\",\"arguments\":{}}"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1); +} + +#[test] +fn fence_ranges_cover_languages_and_unclosed_fences() { + let text = "a\n```rust\nx\n```\nb\n~~~js\ny\n"; + let ranges = fence_ranges(text); + assert_eq!(ranges.len(), 2); + assert_eq!(&text[ranges[0].clone()], "```rust\nx\n```"); + assert_eq!(ranges[1].end, text.len()); + assert!(fence_ranges("```\nplain\n```").is_empty()); + assert!(fence_ranges("```tool_call\n{}\n```").is_empty()); +} + +#[test] +fn names_are_repaired_against_known_tools() { + let outcome = parse_known( + "{\"name\":\"terminal\\\" parameter=\\\"command\",\"arguments\":{\"command\":\"ls\"}}", + &["terminal", "read_file"], + ); + assert_eq!(outcome.calls[0].name, "terminal"); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::NameRepaired { to, .. } if to == "terminal")) + ); + + let outcome = parse_known( + "{\"name\":\"functions.read_file\",\"arguments\":{}}", + &["read_file"], + ); + assert_eq!(outcome.calls[0].name, "read_file"); + let outcome = parse_known( + "{\"name\":\"Read File\",\"arguments\":{}}", + &["read_file"], + ); + assert_eq!(outcome.calls[0].name, "read_file"); + let outcome = parse_known( + "{\"name\":\"raed_file\",\"arguments\":{}}", + &["read_file", "write_file"], + ); + assert_eq!(outcome.calls[0].name, "read_file"); +} + +#[test] +fn an_unknown_name_is_returned_and_flagged() { + let outcome = parse_known( + "{\"name\":\"launch_missiles\",\"arguments\":{}}", + &["read_file"], + ); + assert_eq!(outcome.calls.len(), 1); + assert_eq!(outcome.calls[0].name, "launch_missiles"); + assert!(matches!( + outcome.diagnostics[0], + ParseDiagnostic::UnknownTool { .. } + )); +} + +#[test] +fn malformed_and_unterminated_blocks_are_reported() { + let outcome = parse_known( + "nope and {\"name\":\"x\"", + &[], + ); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::UnterminatedBlock { .. })) + ); +} + +#[test] +fn mixed_grammars_in_one_response_parse_in_source_order() { + let text = concat!( + "{\"name\":\"a\",\"arguments\":{}}", + "v", + "<｜tool▁call▁begin｜>c<｜tool▁sep｜>{}<｜tool▁call▁end｜>", + "[TOOL_CALLS][{\"name\":\"d\",\"arguments\":{}}]" + ); + let (_, calls) = parse(text); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b", "c", "d"]); +} + +// ── helpers kept public for hosts ─────────────────────────────────────────── + +#[test] +fn parse_argument_helpers_cover_string_non_string_and_missing_values() { + assert_eq!( + parse_arguments_value(Some(&serde_json::json!("{\"value\":1}"))), + serde_json::json!({ "value": 1 }) + ); + assert_eq!( + parse_arguments_value(Some(&serde_json::json!("not-json"))), + serde_json::json!({}) + ); + assert_eq!( + parse_arguments_value(Some(&serde_json::json!({ "value": 2 }))), + serde_json::json!({ "value": 2 }) + ); + assert_eq!(parse_arguments_value(None), serde_json::json!({})); +} + +#[test] +fn parse_tool_call_value_supports_function_shape_flat_shape_and_invalid_names() { + let function_shape = serde_json::json!({ + "function": { "name": "shell", "arguments": "{\"command\":\"ls\"}" } + }); + let parsed = parse_tool_call_value(&function_shape).expect("function call should parse"); + assert_eq!(parsed.name, "shell"); + assert_eq!(parsed.arguments, serde_json::json!({ "command": "ls" })); + + let flat_shape = serde_json::json!({ "name": "echo", "arguments": { "value": "hi" } }); + let parsed = parse_tool_call_value(&flat_shape).expect("flat call should parse"); + assert_eq!(parsed.name, "echo"); + + assert!(parse_tool_call_value(&serde_json::json!({ "name": " " })).is_none()); + assert!(parse_tool_call_value(&serde_json::json!({ "function": {} })).is_none()); + assert!(parse_tool_call_value(&serde_json::json!({ "tool": "echo", "args": {} })).is_none()); +} + +#[test] +fn parse_tool_calls_from_json_value_handles_envelopes_arrays_and_singletons() { + let wrapped = serde_json::json!({ + "tool_calls": [ + { "name": "echo", "arguments": { "value": "one" } }, + { "function": { "name": "shell", "arguments": "{\"command\":\"pwd\"}" } } + ], + "content": "assistant text" + }); + let calls = parse_tool_calls_from_json_value(&wrapped); + assert_eq!(calls.len(), 2); + assert_eq!(calls[1].name, "shell"); + + let single = serde_json::json!({ "name": "echo", "arguments": { "value": "three" } }); + assert_eq!(parse_tool_calls_from_json_value(&single).len(), 1); + + // Tagged contexts widen `input` into arguments. + let answer = serde_json::json!({ "name": "Alice", "input": { "value": "hi" } }); + assert_eq!( + parse_tool_calls_from_json_value(&answer)[0].arguments, + serde_json::json!({ "value": "hi" }) + ); +} + +#[test] +fn json_scanners_cover_common_edge_cases() { + let extracted = + extract_first_json_value_with_end(" text {\"ok\":true} trailing ").expect("json"); + assert_eq!(extracted.0, serde_json::json!({ "ok": true })); + assert!(extracted.1 > 0); + assert!(extract_first_json_value_with_end("no json here").is_none()); + + assert_eq!( + strip_leading_close_tags(" hi "), + "hi " + ); + assert_eq!(strip_leading_close_tags("plain"), "plain"); + assert_eq!(strip_leading_close_tags(" ls -la"); + assert!(text.is_empty()); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "shell"); +} + +#[test] +fn glm_parser_covers_json_payloads_invalid_urls_and_plain_commands() { + let calls = parse_glm_style_tool_calls(concat!( + "\n", + "custom/{\"answer\":42}\n", + "shell/url>not-a-url\n", + "shell/command>https://example.com/has space\n", + "shell/command>echo hi\n", + "not a url" + )); + assert_eq!(calls.len(), 3); + assert_eq!(calls[0].0, "custom"); + assert_eq!(calls[0].1, serde_json::json!({"answer": 42})); + assert_eq!( + calls[1].1, + serde_json::json!({"command": "https://example.com/has space"}) + ); + assert_eq!(calls[2].1, serde_json::json!({"command": "echo hi"})); +} + +#[test] +fn glm_helpers_parse_aliases_urls_and_commands() { + assert_eq!(map_glm_tool_alias("browser_open"), "shell"); + assert_eq!(map_glm_tool_alias("http"), "http_request"); + assert_eq!(map_glm_tool_alias("custom_tool"), "custom_tool"); + + assert_eq!( + build_curl_command("https://example.com?q=1"), + Some("curl -s 'https://example.com?q=1'".into()) + ); + assert_eq!( + build_curl_command("https://exa'mple.com"), + Some("curl -s 'https://exa'\\''mple.com'".into()) + ); + assert!(build_curl_command("ftp://example.com").is_none()); + assert!(build_curl_command("https://example.com/has space").is_none()); + + let calls = parse_glm_style_tool_calls( + "browser_open/url>https://example.com\nhttp_request/url>https://api.example.com\nplain text\nhttps://rust-lang.org", + ); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, "shell"); + assert_eq!(calls[1].0, "http_request"); + assert!(parse_glm_style_tool_calls("https://rust-lang.org").is_empty()); +} diff --git a/crates/tinytools-agent/src/parse/test/harmony_mistral.rs b/crates/tinytools-agent/src/parse/test/harmony_mistral.rs new file mode 100644 index 0000000..85af5fb --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/harmony_mistral.rs @@ -0,0 +1,179 @@ +//! gpt-oss Harmony and Mistral `[TOOL_CALLS]`. + +use super::{parse, parse_known}; +use crate::types::{CallSource, ParseDiagnostic}; + +#[test] +fn harmony_commentary_call_parses() { + let response = "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>{\"city\":\"Paris\"}<|call|>"; + let (text, calls) = parse(response); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[0].arguments["city"], "Paris"); + assert_eq!(calls[0].source, CallSource::Harmony); +} + +#[test] +fn harmony_channel_without_target_is_not_a_call() { + let response = "<|channel|>analysis<|message|>thinking hard<|end|>final answer"; + let (text, calls) = parse(response); + assert!(calls.is_empty()); + assert_eq!(text, response); +} + +#[test] +fn harmony_call_with_start_prefix_and_no_terminator_parses_in_batch() { + let response = + "<|start|>assistant<|channel|>commentary to=functions.read<|message|>{\"path\":\"a\"}"; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "read"); +} + +#[test] +fn harmony_channel_with_target_but_no_message_is_not_a_call_in_batch_mode() { + // No `<|message|>` ever arrives, so a batch parse cannot know whether a + // call is coming; the header is left as ordinary text rather than + // guessed at. + let response = "<|channel|>commentary to=functions.read still thinking"; + let (text, calls) = parse(response); + assert!(calls.is_empty()); + assert_eq!(text, response); +} + +#[test] +fn harmony_channel_with_empty_target_is_skipped_and_next_call_found() { + // `to=` with nothing but whitespace after it names no tool, so the + // grammar skips past that channel message and keeps scanning for a + // later one that does. + let response = "<|channel|>commentary to= <|message|>ignored<|end|><|channel|>commentary to=functions.read<|message|>{\"path\":\"a\"}<|call|>tail"; + let (text, calls) = parse(response); + assert_eq!( + text, + "<|channel|>commentary to= <|message|>ignored<|end|>\ntail" + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "read"); + assert_eq!(calls[0].arguments["path"], "a"); +} + +#[test] +fn mistral_v3_array_form_parses() { + let response = + "[TOOL_CALLS] [{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}]"; + let (text, calls) = parse(response); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[0].source, CallSource::Mistral); +} + +#[test] +fn mistral_v11_name_args_form_parses() { + let response = "Sure. [TOOL_CALLS]get_weather[ARGS]{\"city\":\"Paris\"}"; + let (text, calls) = parse(response); + assert_eq!(text, "Sure."); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["city"], "Paris"); +} + +#[test] +fn mistral_v3_array_of_non_call_objects_yields_no_calls() { + // The array parses as JSON but contains no `name`/`arguments` call + // shape, so the v3 branch must not treat it as a found block — it + // falls through to the (here, also empty) v11 scan and the response + // is reported as malformed rather than silently dropped. + let response = "[TOOL_CALLS] [{\"foo\":1}]"; + let outcome = parse_known(response, &[]); + assert!(outcome.calls.is_empty()); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); +} + +#[test] +fn mistral_v11_name_with_invalid_characters_is_not_a_call() { + let response = "[TOOL_CALLS]get-weather[ARGS]{\"city\":\"Paris\"}"; + let outcome = parse_known(response, &[]); + assert!(outcome.calls.is_empty()); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); +} + +#[test] +fn mistral_v11_args_with_unparseable_json_is_not_a_call() { + let response = "[TOOL_CALLS]get_weather[ARGS]not json at all"; + let outcome = parse_known(response, &[]); + assert!(outcome.calls.is_empty()); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); +} + +#[test] +fn mistral_v11_non_object_arguments_are_recovered_into_an_object() { + let response = "[TOOL_CALLS]get_weather[ARGS][1,2,3]"; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert!(calls[0].arguments.is_object()); +} + +#[test] +fn mistral_marker_with_no_parseable_call_is_malformed_in_batch_mode() { + let response = "[TOOL_CALLS] this trails off with no call shape"; + let outcome = parse_known(response, &[]); + assert!(outcome.calls.is_empty()); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); + assert!(outcome.text.contains("this trails off with no call shape")); +} + +#[test] +fn mistral_marker_with_no_call_yet_is_held_while_streaming() { + use crate::stream::StreamScrubber; + + let mut s = StreamScrubber::new(); + let first = s.feed("[TOOL_CALLS]get_wea"); + assert_eq!(first.text, "", "a pending marker must be held"); + // The trailing `!` cannot be part of a v11 tool name, so it disambiguates + // the block as finished; trailing alphanumeric text alone would still be + // ambiguous with a directly-appended continuation call and must be held + // (see `mistral_v11_second_call_split_across_fragments_is_not_lost`). + let second = s.feed("ther[ARGS]{\"city\":\"Paris\"}tail!"); + assert_eq!(second.text, "tail!"); + assert_eq!(second.calls[0].name, "get_weather"); +} + +#[test] +fn mistral_v11_second_call_split_across_fragments_is_not_lost() { + // The v11 form lets a second call follow directly with no fresh + // `[TOOL_CALLS]` marker. A naive streamer that finalizes the block the + // moment the first `NAME[ARGS]{...}` completes drops the second call + // the instant a fragment boundary falls between them. + use crate::stream::StreamScrubber; + + let mut s = StreamScrubber::new(); + let first = s.feed("[TOOL_CALLS]a[ARGS]{\"x\":1}"); + assert!(first.calls.is_empty(), "must hold until disambiguated"); + let second = s.feed("b[ARGS]{\"y\":2} done!"); + assert_eq!(second.calls.len(), 2); + assert_eq!(second.calls[0].name, "a"); + assert_eq!(second.calls[1].name, "b"); + assert_eq!(second.text, " done!"); +} diff --git a/crates/tinytools-agent/src/parse/test/invoke_xml.rs b/crates/tinytools-agent/src/parse/test/invoke_xml.rs new file mode 100644 index 0000000..caffbb1 --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/invoke_xml.rs @@ -0,0 +1,246 @@ +//! `` XML: Claude, `DeepSeek` DSML, namespaced, and `` forms. + +use super::{parse, parse_known}; +use crate::types::CallSource; +use crate::{PFormatRegistry, parse_tool_calls_with_pformat}; + +#[test] +fn claude_invoke_blocks_preserve_typed_parameters() { + let source = concat!( + "before ", + "42", + "true", + " ", + "ignored after" + ); + let (text, calls) = parse(source); + assert_eq!(text, "before\nafter"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "run"); + assert_eq!(calls[0].arguments["number"], 42); + assert_eq!(calls[0].arguments["flag"], true); + assert_eq!(calls[0].arguments["empty"], ""); + assert_eq!(calls[0].source, CallSource::InvokeXml); +} + +#[test] +fn unclosed_invoke_is_kept_as_text() { + let malformed = "lead 1"; + let (text, calls) = parse(malformed); + assert_eq!( + text, + "lead 1" + ); + assert!(calls.is_empty()); +} + +#[test] +fn anthropic_function_calls_wrapper_is_stripped() { + let source = "\n\na.txt\n\n"; + let (text, calls) = parse(source); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["path"], "a.txt"); +} + +#[test] +fn namespaced_invoke_from_muse_spark_parses() { + let source = "echo hi"; + let (text, calls) = parse(source); + assert!(text.is_empty()); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "default.terminal"); + assert_eq!(calls[0].arguments["command"], "echo hi"); +} + +#[test] +fn function_equals_form_with_parameter_children_parses() { + let source = "Paris3"; + let (_, calls) = parse(source); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!( + calls[0].arguments, + serde_json::json!({"city": "Paris", "days": 3}) + ); +} + +#[test] +fn function_equals_form_with_json_body_parses() { + let (_, calls) = parse("{\"city\":\"Paris\"}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["city"], "Paris"); +} + +#[test] +fn gemma_function_name_attribute_form_parses() { + let (_, calls) = + parse("x"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "read"); +} + +// ── DeepSeek DSML ─────────────────────────────────────────────────────────── + +#[test] +fn dsml_parameter_with_arguments_envelope_parses() { + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_FETCH_EMAILS\">\n", + "<｜｜DSML｜｜ parameter name=\"arguments\" string=\"false\">{\"max_results\": 10, \"query\": \"in:inbox\", \"user_id\": \"me\"}\n", + "\n", + "" + ); + let (narrative, calls) = parse(response); + assert!(narrative.is_empty(), "{narrative:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); + assert_eq!(calls[0].arguments["max_results"], 10); + assert_eq!(calls[0].arguments["query"], "in:inbox"); + assert_eq!(calls[0].arguments["user_id"], "me"); +} + +#[test] +fn dsml_invoke_with_direct_json_body_parses() { + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"composio_list_tools\">\n", + "{\"toolkits\":[\"gmail\"]}\n", + "\n", + "" + ); + let (_narrative, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "composio_list_tools"); + assert_eq!(calls[0].arguments["toolkits"], serde_json::json!(["gmail"])); +} + +#[test] +fn dsml_invoke_with_orphan_closing_parameter_tag_parses() { + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_FETCH_EMAILS\">\n", + "{\"label_ids\": [\"INBOX\"], \"ids_only\": true, \"max_results\": 500, \"include_payload\": false, \"verbose\": false}\n", + "\n", + "\n", + "" + ); + let (_narrative, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!( + calls[0].arguments["label_ids"], + serde_json::json!(["INBOX"]) + ); + assert_eq!(calls[0].arguments["max_results"], 500); +} + +#[test] +fn dsml_multiple_invokes_with_empty_args_and_narrative_text_parses() { + let response = concat!( + "I'll verify Gmail access by fetching the profile and listing recent inbox messages.\n\n", + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_GET_PROFILE\">\n\n", + "\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_FETCH_EMAILS\">\n", + "{\"label_ids\": [\"INBOX\"], \"max_results\": 5, \"verbose\": false}\n", + "\n", + "" + ); + let (narrative, calls) = parse(response); + assert_eq!( + narrative, + "I'll verify Gmail access by fetching the profile and listing recent inbox messages." + ); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "GMAIL_GET_PROFILE"); + assert_eq!(calls[0].arguments, serde_json::json!({})); + assert_eq!(calls[1].name, "GMAIL_FETCH_EMAILS"); + assert_eq!(calls[1].arguments["max_results"], 5); +} + +#[test] +fn dsml_parameter_with_named_arguments_parses() { + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"composio_list_tools\">\n", + "<｜｜DSML｜｜ parameter name=\"toolkits\" string=\"true\">[\"twitter\"]\n", + "\n", + "" + ); + let (_narrative, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!( + calls[0].arguments["toolkits"], + serde_json::json!(["twitter"]) + ); +} + +#[test] +fn dsml_mixed_tool_call_closing_tag_parses() { + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_FETCH_EMAILS\">\n", + "{\"label_ids\": [\"INBOX\"], \"max_results\": 2}\n", + "" + ); + let (_narrative, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); + assert_eq!(calls[0].arguments["max_results"], 2); +} + +#[test] +fn dsml_with_pformat_registry_recovers_cleanly() { + let reg = PFormatRegistry::new(); + let response = concat!( + "<｜｜DSML｜｜ calls>\n", + "<｜｜DSML｜｜ invoke name=\"GMAIL_FETCH_EMAILS\">\n", + "<｜｜DSML｜｜ parameter name=\"arguments\">{\"label_ids\": [\"INBOX\"], \"max_results\": 3}\n", + "\n", + "" + ); + let (_narrative, calls) = parse_tool_calls_with_pformat(response, ®); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["max_results"], 3); +} + +#[test] +fn dsml_single_bar_ascii_form_parses() { + let response = "<|DSML|tool_calls><|DSML|invoke name=\"read\">{\"path\":\"/tmp/repro.md\"}"; + let (text, calls) = parse(response); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "read"); + assert_eq!(calls[0].arguments["path"], "/tmp/repro.md"); +} + +#[test] +fn dsml_fullwidth_single_bar_with_string_parameter_keeps_markers_inside_values() { + let response = concat!( + "<｜DSML｜tool_calls><｜DSML｜invoke name=\"message\">", + "<｜DSML｜parameter name=\"text\" string=\"true\">literal <｜DSML｜tool_calls> marker", + "" + ); + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!( + calls[0].arguments["text"], + "literal <｜DSML｜tool_calls> marker" + ); +} + +#[test] +fn dsml_incomplete_invoke_prefix_before_a_valid_invoke_is_ignored() { + let response = "<|DSML|tool_calls>literal <|DSML|invoke marker <|DSML|invoke name=\"read\">{\"path\":\"/tmp/valid.md\"}"; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["path"], "/tmp/valid.md"); +} + +#[test] +fn dsml_markers_never_leak_into_the_narrative() { + let response = "Sure.\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"x\">{}\n\nDone."; + let outcome = parse_known(response, &["x"]); + assert!(!outcome.text.contains("DSML"), "{}", outcome.text); + assert_eq!(outcome.text, "Sure.\nDone."); +} diff --git a/crates/tinytools-agent/src/parse/test/mod.rs b/crates/tinytools-agent/src/parse/test/mod.rs new file mode 100644 index 0000000..da1751d --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/mod.rs @@ -0,0 +1,23 @@ +//! Unit tests for the parse pipeline, one file per grammar plus the engine. +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +mod bare_json; +mod engine; +mod glm; +mod harmony_mistral; +mod invoke_xml; +mod sentinel; +mod tagged; + +use crate::types::{ParseOptions, ParsedToolCall}; + +/// Parses with default options. +pub(super) fn parse(text: &str) -> (String, Vec) { + crate::parse::parse_tool_calls(text) +} + +/// Parses with the given known tools. +pub(super) fn parse_known(text: &str, known: &[&str]) -> crate::types::ParseOutcome { + let known: Vec = known.iter().map(ToString::to_string).collect(); + crate::parse::parse_text(text, &ParseOptions::new().with_known_tools(&known)) +} diff --git a/crates/tinytools-agent/src/parse/test/sentinel.rs b/crates/tinytools-agent/src/parse/test/sentinel.rs new file mode 100644 index 0000000..c7b58fb --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/sentinel.rs @@ -0,0 +1,122 @@ +//! DeepSeek-R1 / V3 and Kimi K2 sentinel tokens. + +use super::{parse, parse_known}; +use crate::types::{CallSource, ParseDiagnostic}; + +#[test] +fn deepseek_r1_function_sep_layout_parses() { + let response = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n```json\n{\"location\": \"Tokyo\"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"; + let (text, calls) = parse(response); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[0].arguments["location"], "Tokyo"); + assert_eq!(calls[0].source, CallSource::Sentinel); +} + +#[test] +fn deepseek_v3_name_sep_layout_parses() { + let (_, calls) = parse( + "<｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>{\"location\":\"Tokyo\"}<｜tool▁call▁end｜>", + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); +} + +#[test] +fn deepseek_delimiters_around_a_json_call_object_parse() { + let text = "prose <｜tool▁call▁begin｜>{\"name\":\"a\",\"arguments\":{\"k\":1}}<｜tool▁call▁end｜> more"; + let (cleaned, calls) = parse(text); + assert_eq!(cleaned, "prose\nmore"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["k"], 1); +} + +#[test] +fn kimi_k2_section_layout_parses() { + let response = "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"city\": \"Tokyo\"}<|tool_call_end|><|tool_calls_section_end|>"; + let (text, calls) = parse(response); + assert!(text.is_empty(), "{text:?}"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[0].arguments["city"], "Tokyo"); +} + +#[test] +fn kimi_k2_parallel_calls_parse_in_order() { + let response = concat!( + "<|tool_calls_section_begin|>", + "<|tool_call_begin|>functions.a:0<|tool_call_argument_begin|>{\"x\":1}<|tool_call_end|>", + "<|tool_call_begin|>functions.b:1<|tool_call_argument_begin|>{\"y\":2}<|tool_call_end|>", + "<|tool_calls_section_end|>" + ); + let (_, calls) = parse(response); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b"]); +} + +#[test] +fn unterminated_sentinel_block_is_kept_as_text() { + let text = "start <｜tool▁call▁begin｜>{\"name\":\"a\""; + let (cleaned, calls) = parse(text); + assert!(calls.is_empty()); + assert_eq!(cleaned, text); +} + +#[test] +fn a_sentinel_block_that_decodes_to_nothing_is_malformed() { + let response = "<|tool_call_begin|>garbage that is not json<|tool_call_end|>"; + let outcome = parse_known(response, &[]); + assert!(outcome.calls.is_empty()); + assert!( + outcome + .diagnostics + .iter() + .any(|d| matches!(d, ParseDiagnostic::MalformedBlock { .. })) + ); +} + +#[test] +fn kimi_argument_begin_with_no_name_before_it_is_not_a_call() { + let response = "<|tool_call_begin|><|tool_call_argument_begin|>{}<|tool_call_end|>"; + let (_, calls) = parse(response); + assert!(calls.is_empty()); +} + +#[test] +fn kimi_name_keeps_a_non_numeric_colon_suffix() { + // The generic post-parse name repair also trims at a bare colon (the + // Kimi index separator `:0`), so a known tool that legitimately + // contains one is required to observe that `kimi_name` itself keeps a + // non-numeric suffix rather than treating it as an index. + let response = + "<|tool_call_begin|>functions.foo:bar<|tool_call_argument_begin|>{}<|tool_call_end|>"; + let outcome = parse_known(response, &["foo:bar"]); + assert_eq!(outcome.calls.len(), 1); + assert_eq!(outcome.calls[0].name, "foo:bar"); +} + +#[test] +fn kimi_call_with_no_arguments_after_the_separator_defaults_to_empty_object() { + let response = "<|tool_call_begin|>functions.read<|tool_call_argument_begin|><|tool_call_end|>"; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "read"); + assert_eq!(calls[0].arguments, serde_json::json!({})); +} + +#[test] +fn deepseek_function_prefixed_name_with_no_newline_still_parses() { + let response = "<|tool_call_begin|>function<|tool_sep|>get_weather<|tool_call_end|>"; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[0].arguments, serde_json::json!({})); +} + +#[test] +fn deepseek_sep_with_an_empty_name_is_not_a_call() { + let response = "<|tool_call_begin|><|tool_sep|>{}<|tool_call_end|>"; + let (_, calls) = parse(response); + assert!(calls.is_empty()); +} diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs new file mode 100644 index 0000000..8820fe8 --- /dev/null +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -0,0 +1,313 @@ +//! `` family: spellings, garbled pipes, fences, Kimi bodies. + +use super::parse; +use crate::parse::grammar::tagged::recover_sentinel_body; +use crate::types::CallSource; +use crate::{PFormatRegistry, PFormatToolParams, parse_tool_calls_with_pformat}; + +#[test] +fn canonical_tag_with_surrounding_prose() { + let xml = "before\n\n{\"name\":\"echo\",\"arguments\":{\"value\":\"two\"}}\n\nafter"; + let (text, calls) = parse(xml); + assert_eq!(text, "before\nafter"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({ "value": "two" })); + assert_eq!(calls[0].source, CallSource::TaggedJson); + assert!(calls[0].id.is_none(), "text calls never carry an id"); +} + +#[test] +fn multiple_calls_keep_prose_between_them() { + let text = r#"a{"name":"one","arguments":{}}b{"name":"two","arguments":{"x":1}}c"#; + let (cleaned, calls) = parse(text); + assert_eq!(cleaned, "a\nb\nc"); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "one"); + assert_eq!(calls[1].name, "two"); +} + +#[test] +fn missing_arguments_default_to_empty_object() { + let (_, calls) = parse(r#"{"name":"noargs"}"#); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({})); +} + +#[test] +fn spelling_variants_and_bare_invoke_literal() { + let (text, calls) = + parse("{\"name\":\"echo\",\"arguments\":{\"value\":\"three\"}}"); + assert!(text.is_empty()); + assert_eq!(calls.len(), 1); + + let (_, calls) = parse("{\"name\":\"a\",\"arguments\":{}}"); + assert_eq!(calls[0].name, "a"); + let (_, calls) = parse("{\"name\":\"b\",\"arguments\":{}}"); + assert_eq!(calls[0].name, "b"); +} + +#[test] +fn attribute_form_and_pipe_variant_open_a_block() { + let (cleaned, calls) = + parse(r#"{"name":"foo","arguments":{"a":1}}"#); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "foo"); + assert!(cleaned.is_empty()); + + let (_, calls) = parse(r#"{"name":"bar","arguments":{}}"#); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "bar"); +} + +#[test] +fn plural_tool_calls_tag_is_not_an_opener() { + let (cleaned, calls) = parse("the key holds them"); + assert!(calls.is_empty()); + assert_eq!(cleaned, "the key holds them"); +} + +#[test] +fn malformed_body_is_dropped_without_dispatching() { + let (text, calls) = parse("before not-json after"); + assert_eq!(text, "before\nafter"); + assert!(calls.is_empty()); +} + +#[test] +fn unclosed_tag_with_balanced_json_still_recovers() { + let (text, calls) = parse("before {\"name\":\"echo\",\"arguments\":{}}"); + assert_eq!(text, "before"); + assert_eq!(calls.len(), 1); +} + +#[test] +fn unclosed_tag_without_json_is_kept_as_text() { + let (text, calls) = parse("before not-json"); + assert_eq!(text, "before not-json"); + assert!(calls.is_empty()); + + let (cleaned, calls) = parse("text {\"name\":\"x\""); + assert!(calls.is_empty()); + assert_eq!(cleaned, "text {\"name\":\"x\""); +} + +#[test] +fn prose_mention_without_closing_angle_is_not_a_tag() { + let (cleaned, calls) = parse("wrap it in \nrest"; + let (text, calls) = parse(hybrid); + assert_eq!(calls.len(), 1); + assert_eq!(text, "rest"); +} + +#[test] +fn a_tag_body_may_carry_its_own_json_fence() { + let text = "\n```json\n{\"name\": \"shell\", \"arguments\": {\"command\": \"ls\"}}\n```\n"; + let (_, calls) = parse(text); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "shell"); +} + +#[test] +fn a_tag_body_may_hold_several_calls() { + let response = r#"{"name":"get_weather","arguments":{"city":"London"}}{"name":"get_time","arguments":{"tz":"UTC"}}"#; + let (_, calls) = parse(response); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "get_weather"); + assert_eq!(calls[1].name, "get_time"); +} + +#[test] +fn a_tagged_body_honours_argument_key_aliases() { + for alias in ["args", "parameters", "params", "input"] { + let text = + format!(r#"{{"name":"shell","{alias}":{{"command":"ls"}}}}"#); + let (_, calls) = parse(&text); + assert_eq!(calls.len(), 1, "{alias}"); + assert_eq!(calls[0].arguments["command"], "ls", "{alias}"); + } +} + +#[test] +fn a_tagged_body_with_relaxed_json_is_repaired() { + let (_, calls) = + parse(r#"{name:"get_weather",arguments:{city:"Paris"}}"#); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["city"], "Paris"); +} + +// ── garbled sentinel pipes leaked into the markers ────────────────────────── + +#[test] +fn garbled_pipe_tags_with_json_body_and_call_prefix_parse() { + let garbled = r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "\"University of Colorado\"", "verbose": true}}"#; + let (_text, calls) = parse(garbled); + assert_eq!(calls.len(), 1, "expected the garbled call to be recovered"); + assert_eq!(calls[0].name, "GMAIL_LIST_THREADS"); + assert_eq!(calls[0].arguments["query"], "\"University of Colorado\""); + assert_eq!(calls[0].arguments["verbose"], true); +} + +#[test] +fn garbled_pipe_tags_recover_multiple_parallel_calls() { + let garbled = concat!( + r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "a"}}"#, + r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "b"}}"#, + ); + let (_t, calls) = parse(garbled); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].arguments["query"], "a"); + assert_eq!(calls[1].arguments["query"], "b"); +} + +#[test] +fn symmetric_pipe_tags_pair_positionally() { + let garbled = r#"<|tool_call|>{"name":"echo","arguments":{"msg":"hi"}}<|tool_call|>"#; + let (_t, calls) = parse(garbled); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["msg"], "hi"); + + let (_t, calls) = parse(r#"{"name":"b","arguments":{}}"#); + assert_eq!(calls.len(), 1); +} + +#[test] +fn pformat_pipes_in_a_body_are_not_garbled_tags() { + let mut registry = PFormatRegistry::new(); + registry.insert( + "get_weather".into(), + PFormatToolParams::from_schema(&serde_json::json!({ + "type": "object", + "properties": { "city": { "type": "string" }, "unit": { "type": "string" } }, + "required": ["city", "unit"] + })), + ); + let (_, calls) = parse_tool_calls_with_pformat( + "get_weather[0|London|1|metric]", + ®istry, + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["city"], "London"); + assert_eq!(calls[0].source, CallSource::PFormat); +} + +// ── Kimi `NAME{…}` bodies (#5119) ─────────────────────────────────────────── + +#[test] +fn kimi_name_brace_body_with_quote_sentinels_parses() { + let garbled = r#"<|tool_call>call:GMAIL_FETCH_EMAILS{label_ids:[<|"|>INBOX<|"|>],max_results:1,verbose:true}"#; + let (_text, calls) = parse(garbled); + assert_eq!(calls.len(), 1, "the garbled Kimi call must be recovered"); + assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); + assert_eq!( + calls[0].arguments["label_ids"], + serde_json::json!(["INBOX"]) + ); + assert_eq!(calls[0].arguments["max_results"], 1); + assert_eq!(calls[0].arguments["verbose"], true); +} + +#[test] +fn kimi_name_brace_body_integer_only_parses() { + let garbled = r"<|tool_call>call:GMAIL_FETCH_EMAILS{max_results:5}"; + let (_text, calls) = parse(garbled); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments["max_results"], 5); +} + +#[test] +fn sentinel_body_recovery_leaves_other_shapes_alone() { + assert!(recover_sentinel_body(r#"{"name":"echo","arguments":{}}"#).is_none()); + assert!(recover_sentinel_body("get_weather[London|metric]").is_none()); + assert!(recover_sentinel_body("FOO{a:1} trailing").is_none()); +} + +// ── P-Format registry interplay ───────────────────────────────────────────── + +fn echo_registry() -> PFormatRegistry { + let mut reg = PFormatRegistry::new(); + reg.insert( + "echo".to_string(), + PFormatToolParams::from_schema(&serde_json::json!({ + "type": "object", + "properties": { "value": { "type": "string" } } + })), + ); + reg +} + +#[test] +fn a_pformat_tag_does_not_suppress_a_sibling_glm_tag() { + let response = + "echo[0|hello]\nshell/command>ls -la"; + let (_narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["echo", "shell"]); +} + +#[test] +fn a_pformat_tag_does_not_suppress_a_sibling_fenced_json_tag() { + let response = "echo[0|hello]\n\n```json\n{\"name\": \"shell\", \"arguments\": {\"command\": \"ls\"}}\n```\n"; + let (_narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); + let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, vec!["echo", "shell"]); +} + +#[test] +fn a_json_body_is_not_double_counted_by_the_glm_fallback() { + let response = "echo[0|hello]\n{\"name\": \"shell\", \"arguments\": {\"command\": \"cat a/b>c\"}}"; + let (_narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); + assert_eq!(calls.iter().filter(|c| c.name == "shell").count(), 1); +} + +#[test] +fn a_tagged_body_with_a_registry_still_honours_argument_key_aliases() { + let response = "echo[0|hello]\n{\"name\": \"shell\", \"args\": {\"command\": \"ls\"}}"; + let (_narrative, calls) = parse_tool_calls_with_pformat(response, &echo_registry()); + let shell = calls + .iter() + .find(|c| c.name == "shell") + .expect("aliased tagged call"); + assert_eq!(shell.arguments["command"], "ls"); +} + +#[test] +fn pformat_registry_with_plain_text_yields_nothing() { + let (text, calls) = parse_tool_calls_with_pformat("plain text", &echo_registry()); + assert_eq!(text, "plain text"); + assert!(calls.is_empty()); +} + +#[test] +fn pformat_call_and_claude_invoke_both_survive() { + let (_, calls) = parse_tool_calls_with_pformat( + "echo[0|hi]y", + &echo_registry(), + ); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].name, "echo"); + assert_eq!(calls[1].name, "other"); + assert_eq!(calls[1].arguments, serde_json::json!({"x": "y"})); +} diff --git a/crates/tinytools-agent/src/parse_test.rs b/crates/tinytools-agent/src/parse_test.rs deleted file mode 100644 index e622b96..0000000 --- a/crates/tinytools-agent/src/parse_test.rs +++ /dev/null @@ -1,639 +0,0 @@ -#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] - -use super::*; - -#[test] -fn parse_argument_helpers_cover_string_non_string_and_missing_values() { - assert_eq!( - parse_arguments_value(Some(&serde_json::json!("{\"value\":1}"))), - serde_json::json!({ "value": 1 }) - ); - assert_eq!( - parse_arguments_value(Some(&serde_json::json!("not-json"))), - serde_json::json!({}) - ); - assert_eq!( - parse_arguments_value(Some(&serde_json::json!({ "value": 2 }))), - serde_json::json!({ "value": 2 }) - ); - assert_eq!(parse_arguments_value(None), serde_json::json!({})); -} - -#[test] -fn parse_tool_call_value_supports_function_shape_flat_shape_and_invalid_names() { - let function_shape = serde_json::json!({ - "function": { - "name": "shell", - "arguments": "{\"command\":\"ls\"}" - } - }); - let parsed = parse_tool_call_value(&function_shape).expect("function call should parse"); - assert_eq!(parsed.name, "shell"); - assert_eq!(parsed.arguments, serde_json::json!({ "command": "ls" })); - - let flat_shape = serde_json::json!({ - "name": "echo", - "arguments": { "value": "hi" } - }); - let parsed = parse_tool_call_value(&flat_shape).expect("flat call should parse"); - assert_eq!(parsed.name, "echo"); - assert_eq!(parsed.arguments, serde_json::json!({ "value": "hi" })); - - assert!(parse_tool_call_value(&serde_json::json!({ "name": " " })).is_none()); - assert!(parse_tool_call_value(&serde_json::json!({ "function": {} })).is_none()); -} - -#[test] -fn parse_tool_call_value_accepts_argument_key_aliases() { - // Correct name but the model used `args`/`parameters` instead of the - // canonical `arguments` — recover the call rather than drop it and burn - // an agent iteration (bug-report-2026-05-26 A3). - let with_args = serde_json::json!({ "name": "echo", "args": { "value": "hi" } }); - let parsed = parse_tool_call_value(&with_args).expect("args alias should parse"); - assert_eq!(parsed.name, "echo"); - assert_eq!(parsed.arguments, serde_json::json!({ "value": "hi" })); - - let with_parameters = serde_json::json!({ - "function": { "name": "shell", "parameters": "{\"command\":\"ls\"}" } - }); - let parsed = parse_tool_call_value(&with_parameters).expect("parameters alias should parse"); - assert_eq!(parsed.name, "shell"); - assert_eq!(parsed.arguments, serde_json::json!({ "command": "ls" })); - - // Name stays strict: an arg alias without a recognized name key is not - // a tool call (guards the whole-response JSON parse path). - assert!(parse_tool_call_value(&serde_json::json!({ "tool": "echo", "args": {} })).is_none()); -} - -#[test] -fn whole_response_singleton_ignores_generic_arg_aliases() { - // A plain JSON answer that happens to carry a `name` plus a generic, - // object-valued `input`. Tagged contexts widen `input` into arguments… - let answer = serde_json::json!({ "name": "Alice", "input": { "value": "hi" } }); - let tagged = parse_tool_calls_from_json_value(&answer); - assert_eq!(tagged.len(), 1); - assert_eq!(tagged[0].arguments, serde_json::json!({ "value": "hi" })); - - // …but the whole-response (bare singleton) path must treat this as plain - // text, not a tool call: it carries no canonical `arguments` marker, only - // a `name` that happens to match a tool (CodeRabbit, #2683). - let whole = parse_tool_calls_from_json_value_aliased(&answer, false); - assert!( - whole.is_empty(), - "bare whole-response object without canonical `arguments` must not dispatch a tool call" - ); - - // A bare object WITH the canonical `arguments` key is still recognized on - // the whole-response path — `arguments` is the explicit tool-call marker. - let bare_call = serde_json::json!({ "name": "echo", "arguments": { "value": "hi" } }); - let calls = parse_tool_calls_from_json_value_aliased(&bare_call, false); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "echo"); - assert_eq!(calls[0].arguments, serde_json::json!({ "value": "hi" })); - - // The `tool_calls`-keyed envelope is an explicit marker and stays - // permissive even when aliases are forbidden for bare objects. - let envelope = serde_json::json!({ - "tool_calls": [ { "name": "echo", "input": { "value": "hi" } } ] - }); - let calls = parse_tool_calls_from_json_value_aliased(&envelope, false); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "echo"); - assert_eq!(calls[0].arguments, serde_json::json!({ "value": "hi" })); -} - -#[test] -fn parse_tool_calls_from_json_value_handles_tool_calls_array_arrays_and_singletons() { - let wrapped = serde_json::json!({ - "tool_calls": [ - { "name": "echo", "arguments": { "value": "one" } }, - { "function": { "name": "shell", "arguments": "{\"command\":\"pwd\"}" } } - ], - "content": "assistant text" - }); - let calls = parse_tool_calls_from_json_value(&wrapped); - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].name, "echo"); - assert_eq!(calls[1].name, "shell"); - - let array = serde_json::json!([ - { "name": "echo", "arguments": { "value": "two" } }, - { "name": " " } - ]); - let calls = parse_tool_calls_from_json_value(&array); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].arguments, serde_json::json!({ "value": "two" })); - - let single = serde_json::json!({ "name": "echo", "arguments": { "value": "three" } }); - let calls = parse_tool_calls_from_json_value(&single); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "echo"); -} - -#[test] -fn tag_and_json_extractors_cover_common_edge_cases() { - assert_eq!( - find_first_tag("hi there", &["", ""]), - Some((3, "")) - ); - assert_eq!( - matching_tool_call_close_tag(""), - Some("") - ); - assert_eq!(matching_tool_call_close_tag(""), None); - - let extracted = extract_first_json_value_with_end(" text {\"ok\":true} trailing ") - .expect("json should be found"); - assert_eq!(extracted.0, serde_json::json!({ "ok": true })); - assert!(extracted.1 > 0); - - assert_eq!( - strip_leading_close_tags(" hi "), - "hi " - ); - assert_eq!(strip_leading_close_tags("plain"), "plain"); - - let values = extract_json_values("before {\"a\":1} [1,2] after"); - assert_eq!( - values, - vec![serde_json::json!({ "a": 1 }), serde_json::json!([1, 2])] - ); - - assert_eq!( - find_json_end(" {\"a\":\"}\"}tail"), - Some(" {\"a\":\"}\"}".len()) - ); - assert_eq!(find_json_end("[1,2,3]"), None); -} - -#[test] -fn invoke_attribute_blocks_preserve_typed_parameters_and_reject_malformed_markup() { - let source = concat!( - "before ", - "42", - "true", - " ", - "ignored after" - ); - let (text, calls) = parse_tool_calls(source); - assert_eq!(text, "before\nafter"); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "run"); - assert_eq!(calls[0].arguments["number"], 42); - assert_eq!(calls[0].arguments["flag"], true); - assert_eq!(calls[0].arguments["empty"], ""); - - let malformed = "lead 1"; - let (text, calls) = parse_tool_calls(malformed); - assert_eq!( - text, - "lead\n1" - ); - assert!(calls.is_empty()); - assert!(find_invoke_attr_tag("plain ").is_some()); -} - -#[test] -fn extraction_helpers_cover_invalid_and_incomplete_inputs() { - assert!(extract_first_json_value_with_end("no json here").is_none()); - assert_eq!(strip_leading_close_tags(" not-a-url\n", - "shell/command>https://example.com/has space\n", - "shell/command>echo hi\n", - "not a url" - )); - assert_eq!(calls.len(), 3); - assert_eq!(calls[0].0, "custom"); - assert_eq!(calls[0].1, serde_json::json!({"answer": 42})); - assert_eq!( - calls[1].1, - serde_json::json!({"command": "https://example.com/has space"}) - ); - assert_eq!(calls[2].1, serde_json::json!({"command": "echo hi"})); -} - -#[test] -fn parser_keeps_unclosed_and_malformed_tag_text_without_dispatching_it() { - let malformed = "before not-json after"; - let (text, calls) = parse_tool_calls(malformed); - assert_eq!(text, "before\nafter"); - assert!(calls.is_empty()); - - let unclosed = "before {\"name\":\"echo\",\"arguments\":{}}"; - let (text, calls) = parse_tool_calls(unclosed); - assert_eq!(text, "before"); - assert_eq!(calls.len(), 1); - - let missing_close = "before not-json"; - let (text, calls) = parse_tool_calls(missing_close); - assert_eq!(text, "before\nnot-json"); - assert!(calls.is_empty()); -} - -#[test] -fn pformat_wrapper_returns_canonical_result_when_no_positional_call_is_recovered() { - let empty = PFormatRegistry::new(); - let (text, calls) = parse_tool_calls_with_pformat("shell/command>echo hi", &empty); - assert!(text.is_empty()); - assert_eq!(calls.len(), 1); - - let mut registry = PFormatRegistry::new(); - registry.insert( - "echo".into(), - PFormatToolParams::from_schema(&serde_json::json!({"type":"object"})), - ); - let (text, calls) = parse_tool_calls_with_pformat("plain text", ®istry); - assert_eq!(text, "plain text"); - assert!(calls.is_empty()); -} - -#[test] -fn pformat_wrapper_retains_a_claude_invoke_alongside_a_positional_call() { - let mut registry = PFormatRegistry::new(); - registry.insert( - "echo".into(), - PFormatToolParams::from_schema(&serde_json::json!({ - "type": "object", - "properties": { "value": { "type": "string" } } - })), - ); - let (_, calls) = parse_tool_calls_with_pformat( - concat!( - "echo[0|hi]", - "y" - ), - ®istry, - ); - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].name, "echo"); - assert_eq!(calls[1].name, "other"); - assert_eq!(calls[1].arguments, serde_json::json!({"x": "y"})); -} - -#[test] -fn glm_helpers_parse_aliases_urls_and_commands() { - assert_eq!(map_glm_tool_alias("browser_open"), "shell"); - assert_eq!(map_glm_tool_alias("http"), "http_request"); - assert_eq!(map_glm_tool_alias("custom_tool"), "custom_tool"); - - assert_eq!( - build_curl_command("https://example.com?q=1"), - Some("curl -s 'https://example.com?q=1'".into()) - ); - assert_eq!( - build_curl_command("https://exa'mple.com"), - Some("curl -s 'https://exa'\\''mple.com'".into()) - ); - assert!(build_curl_command("ftp://example.com").is_none()); - assert!(build_curl_command("https://example.com/has space").is_none()); - - let calls = parse_glm_style_tool_calls( - "browser_open/url>https://example.com\nhttp_request/url>https://api.example.com\nplain text\nhttps://rust-lang.org", - ); - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].0, "shell"); - assert_eq!(calls[1].0, "http_request"); - assert!(parse_glm_style_tool_calls("https://rust-lang.org").is_empty()); -} - -#[test] -fn parse_tool_calls_supports_native_json_xml_markdown_and_glm_formats() { - let native = serde_json::json!({ - "content": "native text", - "tool_calls": [ - { "name": "echo", "arguments": { "value": "one" } } - ] - }) - .to_string(); - let (text, calls) = parse_tool_calls(&native); - assert_eq!(text, "native text"); - assert_eq!(calls.len(), 1); - - let xml = "before\n\n{\"name\":\"echo\",\"arguments\":{\"value\":\"two\"}}\n\nafter"; - let (text, calls) = parse_tool_calls(xml); - assert_eq!(text, "before\nafter"); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].arguments, serde_json::json!({ "value": "two" })); - - let unclosed = "{\"name\":\"echo\",\"arguments\":{\"value\":\"three\"}}"; - let (text, calls) = parse_tool_calls(unclosed); - assert!(text.is_empty()); - assert_eq!(calls.len(), 1); - - let markdown = - "lead\n```tool_call\n{\"name\":\"echo\",\"arguments\":{\"value\":\"four\"}}\n```\ntrail"; - let (text, calls) = parse_tool_calls(markdown); - assert_eq!(text, "lead\ntrail"); - assert_eq!(calls.len(), 1); - - let glm = "shell/command>ls -la"; - let (text, calls) = parse_tool_calls(glm); - assert!(text.is_empty()); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "shell"); -} - -// ── lenient recovery of pipe-garbled tags ──────────────────────── - -#[test] -fn garbled_pipe_tags_with_json_body_and_call_prefix_parse() { - // Exact shape seen from a small Composio sub-agent: native `<|…|>` sentinel - // pipes leaked into the tags (`<|tool_call>` / ``) and the body - // is prefixed with `call:`. Without the normalizer this drops silently and - // the tool never runs; with it, the real call is recovered. - let garbled = r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "\"University of Colorado\"", "verbose": true}}"#; - let (_text, calls) = parse_tool_calls(garbled); - assert_eq!(calls.len(), 1, "expected the garbled call to be recovered"); - assert_eq!(calls[0].name, "GMAIL_LIST_THREADS"); - assert_eq!(calls[0].arguments["query"], "\"University of Colorado\""); - assert_eq!(calls[0].arguments["verbose"], true); -} - -#[test] -fn garbled_pipe_tags_recover_multiple_parallel_calls() { - let garbled = concat!( - r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "a"}}"#, - r#"<|tool_call>call:{"name": "GMAIL_LIST_THREADS", "arguments": {"query": "b"}}"#, - ); - let (_t, calls) = parse_tool_calls(garbled); - assert_eq!(calls.len(), 2); - assert_eq!(calls[0].arguments["query"], "a"); - assert_eq!(calls[1].arguments["query"], "b"); -} - -#[test] -fn normalize_leaves_clean_output_untouched() { - // No piped marker → cheap Borrowed no-op, and a canonical call still parses. - let clean = r#"{"name":"echo","arguments":{}}"#; - assert!(matches!( - normalize_garbled_tool_call_tags(clean), - std::borrow::Cow::Borrowed(_) - )); - let (_t, calls) = parse_tool_calls(clean); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "echo"); -} - -#[test] -fn normalize_repairs_open_and_close_pipe_variants() { - // Leading pipe → open; trailing pipe (no slash) → close. - assert_eq!( - normalize_garbled_tool_call_tags("<|tool_call>BODY").as_ref(), - "BODY" - ); - // Slash-bearing close variants normalize too. - assert_eq!( - normalize_garbled_tool_call_tags("b").as_ref(), - "b" - ); -} - -#[test] -fn normalize_pairs_symmetric_both_pipe_tags() { - // `<|tool_call|>` on BOTH sides — a hardcoded open/close map can't tell them - // apart; positional pairing does (1st = open, 2nd = close). - let garbled = r#"<|tool_call|>{"name":"echo","arguments":{"msg":"hi"}}<|tool_call|>"#; - let (_t, calls) = parse_tool_calls(garbled); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "echo"); - assert_eq!(calls[0].arguments["msg"], "hi"); -} - -#[test] -fn normalize_leaves_pformat_pipe_args_untouched() { - // Pipes appear in P-Format BODIES (`name[a|b]`), not the tags — the - // garbled-tag guard must not touch a well-formed positional call. - let clean = "get_weather[London|metric]"; - assert!(matches!( - normalize_garbled_tool_call_tags(clean), - std::borrow::Cow::Borrowed(_) - )); -} - -// ── #5119: recover the Kimi `NAME{…}` argument-sentinel body ───────────────── - -#[test] -fn garbled_kimi_name_brace_body_with_quote_sentinels_parses() { - // The EXACT shape observed from `integrations_agent`/`burst-v1` (Kimi-K2) on - // the post-contract retry: garbled tags PLUS a `NAME{…}` body with unquoted - // keys and the `<|"|>` argument-quote sentinel around string values. Before - // the recovery this parsed to zero tool calls (the tag fix alone left an - // unparseable body), so GMAIL_FETCH_EMAILS never ran and the turn looped. - let garbled = r#"<|tool_call>call:GMAIL_FETCH_EMAILS{label_ids:[<|"|>INBOX<|"|>],max_results:1,verbose:true}"#; - let (_text, calls) = parse_tool_calls(garbled); - assert_eq!(calls.len(), 1, "the garbled Kimi call must be recovered"); - assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); - assert_eq!( - calls[0].arguments["label_ids"], - serde_json::json!(["INBOX"]) - ); - assert_eq!(calls[0].arguments["max_results"], 1); - assert_eq!(calls[0].arguments["verbose"], true); -} - -#[test] -fn garbled_kimi_name_brace_body_integer_only_parses() { - // The integer-only variant (no string values → no `<|"|>` sentinel, but the - // body is still the unparseable `NAME{unquoted-keys}` shape). Observed as - // `{max_results:5}` on the staging repro. - let garbled = r"<|tool_call>call:GMAIL_FETCH_EMAILS{max_results:5}"; - let (_text, calls) = parse_tool_calls(garbled); - assert_eq!(calls.len(), 1); - assert_eq!(calls[0].name, "GMAIL_FETCH_EMAILS"); - assert_eq!(calls[0].arguments["max_results"], 5); -} - -#[test] -fn recover_sentinel_body_leaves_canonical_and_pformat_untouched() { - // Canonical JSON body → empty leading name → not our shape → None. - assert!(recover_sentinel_tool_call_body(r#"{"name":"echo","arguments":{}}"#).is_none()); - // P-Format body (`NAME[…]`, no brace) → None. - assert!(recover_sentinel_tool_call_body("get_weather[London|metric]").is_none()); - // Trailing garbage after the object → not a clean `NAME{…}` → None. - assert!(recover_sentinel_tool_call_body("FOO{a:1} trailing").is_none()); -} - -#[test] -fn quote_bare_json_object_keys_respects_string_values() { - // A `,ident:` sequence INSIDE a string value must not be quoted; only - // structural keys after `{`/`,` are rewritten. - let out = quote_bare_json_object_keys(r#"{query:"from:john,to:x",n:1}"#); - assert_eq!(out, r#"{"query":"from:john,to:x","n":1}"#); - // Parses as strict JSON with the value preserved verbatim. - let v: serde_json::Value = serde_json::from_str(&out).unwrap(); - assert_eq!(v["query"], "from:john,to:x"); - assert_eq!(v["n"], 1); -} - -#[test] -fn quote_bare_json_object_keys_leaves_array_literals_unquoted() { - // Bare literals inside arrays must not be quoted. A comma inside an array - // should not trigger `expect_key = true` because arrays do not have keys. - let out = quote_bare_json_object_keys(r"{flags:[true,false],n:null}"); - assert_eq!(out, r#"{"flags":[true,false],"n":null}"#); - // Parses as strict JSON with the values preserved as booleans and null. - let v: serde_json::Value = serde_json::from_str(&out).unwrap(); - assert_eq!(v["flags"], serde_json::json!([true, false])); - assert_eq!(v["n"], serde_json::Value::Null); -} - -#[test] -fn parse_tool_calls_with_pformat_preserves_multi_call_tag_bodies() { - // A single tag body can hold multiple JSON calls (e.g., two - // adjacent objects or a {"tool_calls":[...]} envelope). The ordinal pairing - // must not drop them when re-parsing the tag body. - use crate::PFormatRegistry; - - let registry = PFormatRegistry::new(); - let response = r#"{"name":"get_weather","arguments":{"city":"London"}}{"name":"get_time","arguments":{"tz":"UTC"}}"#; - let (_, calls) = parse_tool_calls_with_pformat(response, ®istry); - - assert_eq!( - calls.len(), - 2, - "both JSON calls in the tag body must be recovered" - ); - assert_eq!(calls[0].name, "get_weather"); - assert_eq!(calls[0].arguments["city"], "London"); - assert_eq!(calls[1].name, "get_time"); - assert_eq!(calls[1].arguments["tz"], "UTC"); -} - -// ── Regression probe: mixed p-format + non-JSON tags ───────────────────────── - -use crate::{PFormatRegistry, PFormatToolParams}; - -/// A p-format tag alongside a GLM-style sibling. -/// -/// Once any tag yields a p-format call, the walk stops falling back to the -/// canonical parse, so every remaining tag is on its own. A GLM body -/// (`shell/command>ls -la`) is not JSON, so before the GLM fallback existed -/// this call was silently dropped — the agent lost a tool invocation it had -/// asked for and nothing reported it. -#[test] -fn a_pformat_tag_does_not_suppress_a_sibling_glm_tag() { - let mut reg = PFormatRegistry::new(); - reg.insert( - "echo".to_string(), - PFormatToolParams::from_schema(&serde_json::json!({ - "type": "object", - "properties": { "value": { "type": "string" } } - })), - ); - - let response = concat!( - "echo[0|hello]\n", - "shell/command>ls -la" - ); - let (_narrative, calls) = parse_tool_calls_with_pformat(response, ®); - let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); - - assert!( - names.contains(&"echo"), - "the p-format call must survive: {names:?}" - ); - assert_eq!( - calls.len(), - 2, - "the sibling non-JSON tag was dropped — got {names:?}" - ); -} - -/// The same shape, but the sibling body is JSON inside a markdown fence. -#[test] -fn a_pformat_tag_does_not_suppress_a_sibling_fenced_json_tag() { - let mut reg = PFormatRegistry::new(); - reg.insert( - "echo".to_string(), - PFormatToolParams::from_schema(&serde_json::json!({ - "type": "object", - "properties": { "value": { "type": "string" } } - })), - ); - - let response = concat!( - "echo[0|hello]\n", - "\n```json\n{\"name\": \"shell\", \"arguments\": {\"command\": \"ls\"}}\n```\n" - ); - let (_narrative, calls) = parse_tool_calls_with_pformat(response, ®); - let names: Vec<&str> = calls.iter().map(|c| c.name.as_str()).collect(); - - assert!(names.contains(&"echo"), "p-format call survives: {names:?}"); - assert!( - names.contains(&"shell"), - "the fenced-JSON sibling was dropped — got {names:?}" - ); -} - -/// A JSON body that ALSO looks like GLM's `name/key>value` grammar must not -/// yield the call twice. -/// -/// The GLM fallback runs only when the JSON path found nothing, and this is -/// what pins that ordering. If it ever ran unconditionally, a body containing -/// a `/` and a `>` inside a string value would be counted once as JSON and -/// again as GLM — the agent would execute the same tool twice. -#[test] -fn a_json_body_is_not_double_counted_by_the_glm_fallback() { - let mut reg = PFormatRegistry::new(); - reg.insert( - "echo".to_string(), - PFormatToolParams::from_schema(&serde_json::json!({ - "type": "object", - "properties": { "value": { "type": "string" } } - })), - ); - - let response = concat!( - "echo[0|hello]\n", - "{\"name\": \"shell\", \"arguments\": {\"command\": \"cat a/b>c\"}}" - ); - let (_narrative, calls) = parse_tool_calls_with_pformat(response, ®); - let shell_calls = calls.iter().filter(|c| c.name == "shell").count(); - assert_eq!( - shell_calls, - 1, - "the JSON body was counted twice: {:?}", - calls.iter().map(|c| c.name.as_str()).collect::>() - ); -} - -/// A tagged body may use the argument-key aliases. -/// -/// A `` tag is an explicit tool-call marker, so `args` / -/// `parameters` / `input` are honoured inside it — unlike a bare top-level -/// object, where they are refused so a plain JSON answer cannot read as a -/// call. This pins that the tag path keeps the permissive behaviour: routing -/// it through `parse_tool_calls` instead would silently drop this call. -#[test] -fn a_tagged_body_still_honours_argument_key_aliases() { - let mut reg = PFormatRegistry::new(); - reg.insert( - "echo".to_string(), - PFormatToolParams::from_schema(&serde_json::json!({ - "type": "object", - "properties": { "value": { "type": "string" } } - })), - ); - - let response = concat!( - "echo[0|hello]\n", - "{\"name\": \"shell\", \"args\": {\"command\": \"ls\"}}" - ); - let (_narrative, calls) = parse_tool_calls_with_pformat(response, ®); - let shell = calls - .iter() - .find(|c| c.name == "shell") - .expect("the aliased tagged call must survive"); - assert_eq!(shell.arguments["command"], "ls"); -} diff --git a/crates/tinytools-agent/src/render/calls.rs b/crates/tinytools-agent/src/render/calls.rs new file mode 100644 index 0000000..8ccdaca --- /dev/null +++ b/crates/tinytools-agent/src/render/calls.rs @@ -0,0 +1,32 @@ +//! Rendering a structured call back into the text form a prompt-guided model +//! is shown on replay. +//! +//! A model without a native tool channel cannot read an assistant turn's +//! `tool_calls` field, so on the next request the calls it made are written +//! back into its transcript as the same `` markup it was taught. +//! This is the one place that markup is *written*, so the replay can never +//! drift from what [`crate::parse`] reads. + +use std::fmt::Write as _; + +use serde_json::Value; + +/// One call as `{"name":…,"arguments":…}`. +#[must_use] +pub fn render_json_call(name: &str, arguments: &Value) -> String { + let body = serde_json::json!({ "name": name, "arguments": arguments }); + format!( + "{}", + serde_json::to_string(&body).unwrap_or_else(|_| "{}".to_string()) + ) +} + +/// Several calls, one per line. +#[must_use] +pub fn render_json_calls<'a>(calls: impl IntoIterator) -> String { + let mut out = String::new(); + for (name, arguments) in calls { + let _ = writeln!(out, "{}", render_json_call(name, arguments)); + } + out +} diff --git a/crates/tinytools-agent/src/dialect/catalogue.rs b/crates/tinytools-agent/src/render/catalogue.rs similarity index 100% rename from crates/tinytools-agent/src/dialect/catalogue.rs rename to crates/tinytools-agent/src/render/catalogue.rs diff --git a/crates/tinytools-agent/src/render/instructions.rs b/crates/tinytools-agent/src/render/instructions.rs new file mode 100644 index 0000000..9438016 --- /dev/null +++ b/crates/tinytools-agent/src/render/instructions.rs @@ -0,0 +1,80 @@ +//! The protocol block each dialect puts in the system prompt. +//! +//! One place, so the wording a model reads and the grammar the parser +//! expects cannot drift apart. The JSON block embeds its catalogue because the +//! schemas *are* the protocol for a model writing argument names by hand; the +//! P-Format block does not, because its signatures live in the prompt's tool +//! section next to the descriptions; the native block carries no catalogue at +//! all, because the request does. + +use tinytools::ToolSpec; + +use super::catalogue::render_json_catalogue; + +/// The JSON-in-tag protocol block plus the full-schema catalogue. +#[must_use] +pub fn json_instructions(tools: &[ToolSpec]) -> String { + let mut out = String::new(); + out.push_str("## Tool Use Protocol\n\n"); + out.push_str("To use a tool, wrap a JSON object in tags:\n\n"); + out.push_str( + "```\n\n{\"name\": \"tool_name\", \"arguments\": {\"param\": \"value\"}}\n\n```\n\n", + ); + out.push_str("You may emit multiple tool calls in a single response. "); + out.push_str("After execution, results appear in tags. "); + out.push_str("Continue reasoning with the results until you can give a final answer.\n\n"); + out.push_str("### Available Tools\n\n"); + out.push_str(&render_json_catalogue(tools)); + out +} + +/// The P-Format protocol block — protocol only, no catalogue. +#[must_use] +pub fn pformat_instructions() -> String { + let mut out = String::new(); + out.push_str("## Tool Use Protocol\n\n"); + out.push_str( + "Tool calls use **P-Format** (Parameter-Format): compact, slot-indexed, \ + pipe-delimited syntax wrapped in `` tags. ~80% cheaper on tokens \ + than JSON.\n\n", + ); + out.push_str("```\n\nget_weather[0|London|1|metric]\n\n```\n\n"); + out.push_str( + "**Rules:**\n\ + - Form: `name[index|value|index|value|...]`. Each value is preceded by the slot \ + number it fills, taken from that tool's `Call as:` signature in the `## Tools` \ + section above.\n\ + - **Send only the arguments you mean to send.** To pass just the third slot, \ + write `name[2|value]` — there are no empty slots to count.\n\ + - The signature shows each slot as `index|`, e.g. \ + `search[0||1|]`. `` is a placeholder: replace it with the \ + value, and do not send the name itself.\n\ + - Empty calls: `name[]` for zero-arg tools, or for a call sending no arguments.\n\ + - A call whose indices are missing, non-numeric, or not in the signature is \ + **rejected** — it will not run. Copy the numbers from the signature.\n\ + - Escapes inside argument values: `\\|` → `|`, `\\]` → `]`, `\\\\` → `\\`.\n\ + - You may emit multiple `` blocks in a single response. Each tag holds \ + exactly one call.\n\ + - After tool execution, results appear in `` tags. Continue reasoning \ + with the results until you can give a final answer.\n\ + - If you genuinely need a complex nested argument that p-format can't express, \ + you may fall back to the JSON form: \ + `{\"name\":\"...\",\"arguments\":{...}}`. Prefer p-format \ + for everything else.\n\n", + ); + out +} + +/// The native protocol block: behavioural guidance only. +#[must_use] +pub fn native_instructions() -> String { + [ + "## Tool Use Protocol", + "", + "When a tool is needed, emit tool calls directly via the model's native tool-calling output.", + "Do not only narrate intent (for example, avoid \"Let me check...\") without emitting the tool call.", + "After tool results are provided, continue reasoning and then produce the final answer.", + "", + ] + .join("\n") +} diff --git a/crates/tinytools-agent/src/render/mod.rs b/crates/tinytools-agent/src/render/mod.rs new file mode 100644 index 0000000..f63562a --- /dev/null +++ b/crates/tinytools-agent/src/render/mod.rs @@ -0,0 +1,26 @@ +//! Everything the model *reads*: the catalogue, the protocol block, and the +//! envelope tool results come back in. +//! +//! Rendering and parsing are two faces of one protocol, and this module is +//! the rendering face for every dialect. A catalogue that advertises one +//! grammar next to a parser that expects another is a silent whole-turn +//! failure, so the protocol text lives here, once, and +//! [`crate::dialect`] only chooses which block to use. +//! +//! * [`catalogue`] — the tool list, as signatures or as full schemas; +//! * [`instructions`] — the protocol block for each dialect; +//! * [`calls`] — a structured call written back as `` markup for +//! replay to a prompt-guided model; +//! * [`results`] — the `` envelope and transcript replay for +//! the text dialects, with the boundary-integrity rules that keep a tool +//! output from forging protocol structure. + +pub mod calls; +pub mod catalogue; +pub mod instructions; +pub mod results; + +pub use calls::{render_json_call, render_json_calls}; +pub use catalogue::{CATALOGUE_HEADING, render_json_catalogue, render_pformat_catalogue}; +pub use instructions::{json_instructions, native_instructions, pformat_instructions}; +pub use results::{TOOL_RESULTS_PREFIX, format_results, to_provider_messages}; diff --git a/crates/tinytools-agent/src/dialect/text.rs b/crates/tinytools-agent/src/render/results.rs similarity index 95% rename from crates/tinytools-agent/src/dialect/text.rs rename to crates/tinytools-agent/src/render/results.rs index 42cb8e7..9aa6e09 100644 --- a/crates/tinytools-agent/src/dialect/text.rs +++ b/crates/tinytools-agent/src/render/results.rs @@ -17,8 +17,8 @@ //! it early (CWE-74). The two get different rules because they land in //! different places, and the difference is the point: //! -//! * a **name** is an attribute value, so [`escape_attribute`] escapes the lot; -//! * an **output** is the body, so [`neutralize_protocol_tags`] rewrites only +//! * a **name** is an attribute value, so `escape_attribute` escapes the lot; +//! * an **output** is the body, so `neutralize_protocol_tags` rewrites only //! the protocol tag openers and lets everything else through byte-for-byte. //! //! Escaping the body wholesale is the obvious implementation and the wrong one: @@ -29,7 +29,7 @@ use std::borrow::Cow; use std::fmt::Write as _; -use super::types::{DialectMessage, ToolOutcome, ToolResultEntry, TranscriptEntry}; +use crate::dialect::{DialectMessage, ToolOutcome, ToolResultEntry, TranscriptEntry}; /// Prefix of the synthetic user turn that carries tool results back to a /// text-mode model. @@ -162,10 +162,11 @@ fn neutralize_protocol_tags(value: &str) -> Cow<'_, str> { /// /// Both the name and the output are tool-controlled, and each gets the rule /// that fits where it lands: the name is an attribute value, so it is fully -/// escaped ([`escape_attribute`]); the output is the body, so only protocol -/// tag openers are neutralized ([`neutralize_protocol_tags`]) and the rest +/// escaped (`escape_attribute`); the output is the body, so only protocol +/// tag openers are neutralized (`neutralize_protocol_tags`) and the rest /// reaches the model byte-for-byte. -pub(super) fn format_results(results: &[ToolOutcome]) -> Vec { +#[must_use] +pub fn format_results(results: &[ToolOutcome]) -> Vec { // The overwhelmingly common shape: nothing marked, one framed batch. Kept as // its own branch so an unmarked round allocates exactly what it always did. if !results.iter().any(|result| result.trusted_verbatim) { @@ -236,12 +237,13 @@ fn frame_batch(results: &[ToolOutcome]) -> String { /// /// `text` must be the **raw** model response the text dialect originally /// parsed (tags and all), not the narrative-only text -/// [`ToolDialect::parse_response`](super::ToolDialect::parse_response) returns +/// [`ToolDialect::parse_response`](crate::dialect::ToolDialect::parse_response) returns /// with the `` tags stripped. A host that persists the parsed /// narrative into `TranscriptEntry::AssistantToolCalls::text` replays an /// assistant turn with no visible call, followed by a `` turn /// that answers nothing the model can see. -pub(super) fn to_provider_messages(history: &[TranscriptEntry]) -> Vec { +#[must_use] +pub fn to_provider_messages(history: &[TranscriptEntry]) -> Vec { history .iter() .flat_map(|entry| match entry { diff --git a/crates/tinytools-agent/src/repair/args.rs b/crates/tinytools-agent/src/repair/args.rs new file mode 100644 index 0000000..801d620 --- /dev/null +++ b/crates/tinytools-agent/src/repair/args.rs @@ -0,0 +1,199 @@ +//! Bringing a model's argument value into the shape the tool's schema wants. +//! +//! Three families of defect, each a real capture from a local model: +//! +//! * the arguments arrive as a JSON **string** (sometimes fenced) instead of +//! an object — `"{\"city\":\"Paris\"}"`; +//! * the real object is buried one level down inside an envelope the model +//! invented — `{"properties":{"city":"Paris"}}` (a schema echo), +//! `{"param":{"city":"Paris"}}`; +//! * scalars are the wrong JSON type for the schema — `"42"` for an integer, +//! `"true"` for a boolean, `"[1,2]"` for an array. +//! +//! Everything here is pure and schema-driven. The host still decides whether +//! to *run* a repaired call; this module only makes the repair possible. + +use serde_json::{Map, Value}; + +/// Object keys that may carry the tool **arguments** inside a call object, +/// in priority order. `arguments` is canonical; the rest are what a model +/// drifts to when it copies the schema vocabulary. +pub const ARGUMENT_KEYS: &[&str] = &["arguments", "args", "parameters", "params", "input"]; + +/// Keys under which a model buries the real arguments object one level down. +/// `properties` is the JSON-Schema echo; the rest are invented wrappers. +pub const WRAPPER_KEYS: &[&str] = &[ + "properties", + "arguments", + "args", + "parameters", + "params", + "param", + "input", +]; + +/// Decodes an argument value that may be a stringified (and possibly fenced) +/// JSON document. Non-string values are cloned; an undecodable string becomes +/// an empty object; a missing value is an empty object. +#[must_use] +pub fn decode(raw: Option<&Value>) -> Value { + match raw { + Some(Value::String(s)) => { + let candidate = super::json::strip_code_fence(s); + serde_json::from_str::(candidate) + .ok() + .or_else(|| super::json::recover_object(candidate)) + .unwrap_or_else(|| Value::Object(Map::new())) + } + Some(value) => value.clone(), + None => Value::Object(Map::new()), + } +} + +/// The arguments under the first present [`ARGUMENT_KEYS`] entry of a call +/// object, decoded. Empty object when none is present. +#[must_use] +pub fn from_call_object(call: &Value) -> Value { + for key in ARGUMENT_KEYS { + if let Some(value) = call.get(*key) { + return decode(Some(value)); + } + } + decode(None) +} + +/// Recovers arguments a model buried one level deep inside an envelope. +/// +/// For each [`WRAPPER_KEYS`] entry the rewrite applies only when the tool does +/// not itself declare a parameter of that name (for such a tool the key is +/// data, not an envelope) and when `is_valid` accepts the unwrapped value. +/// Returns the unwrapped value, or `None` when no candidate satisfies both — +/// the caller keeps the original so the model sees a precise validation error +/// rather than a rewritten one. +#[must_use] +pub fn unwrap_envelope( + arguments: &Value, + schema: &Value, + is_valid: &dyn Fn(&Value) -> bool, +) -> Option { + let declared = schema.get("properties").and_then(Value::as_object); + for key in WRAPPER_KEYS { + if declared.is_some_and(|declared| declared.contains_key(*key)) { + continue; + } + let Some(inner) = arguments.get(*key).filter(|inner| inner.is_object()) else { + continue; + }; + if is_valid(inner) { + return Some(inner.clone()); + } + } + None +} + +/// Whether `schema` can accept an object at all. +#[must_use] +pub fn accepts_object(schema: &Value) -> bool { + schema.get("type").is_some_and(|kind| { + kind.as_str() == Some("object") + || kind + .as_array() + .is_some_and(|kinds| kinds.iter().any(|kind| kind.as_str() == Some("object"))) + }) || schema.get("properties").is_some() + || schema.get("required").is_some() + || schema + .get("enum") + .and_then(Value::as_array) + .is_some_and(|values| values.iter().any(Value::is_object)) +} + +/// Coerces string scalars to the primitive type each schema property declares. +/// +/// `"42"` → `42` for `integer`, `"3.5"` → `3.5` for `number`, `"true"` → +/// `true` for `boolean`, a JSON-encoded string → the decoded value for `array` +/// / `object`, and a bare scalar → `[scalar]` for `array`. A value that does +/// not convert is left as it was, so the schema validator still reports it. +/// Recurses into nested objects and array items with their own schemas. +#[must_use] +pub fn coerce_to_schema(arguments: Value, schema: &Value) -> Value { + let Value::Object(map) = arguments else { + return arguments; + }; + let Some(properties) = schema.get("properties").and_then(Value::as_object) else { + return Value::Object(map); + }; + let mut out = Map::with_capacity(map.len()); + for (key, value) in map { + let coerced = match properties.get(&key) { + Some(property) => coerce_value(value, property), + None => value, + }; + out.insert(key, coerced); + } + Value::Object(out) +} + +fn schema_type(schema: &Value) -> Option<&str> { + match schema.get("type") { + Some(Value::String(s)) => Some(s.as_str()), + Some(Value::Array(items)) => items + .iter() + .find_map(|v| v.as_str().filter(|s| *s != "null")), + _ => None, + } +} + +fn coerce_value(value: Value, schema: &Value) -> Value { + match (schema_type(schema), value) { + (Some("integer"), Value::String(s)) => s + .trim() + .parse::() + .map_or_else(|_| Value::String(s), Value::from), + (Some("number"), Value::String(s)) => s + .trim() + .parse::() + .ok() + .and_then(|n| serde_json::Number::from_f64(n).map(Value::Number)) + .unwrap_or(Value::String(s)), + (Some("boolean"), Value::String(s)) => match s.trim() { + "true" | "True" | "TRUE" => Value::Bool(true), + "false" | "False" | "FALSE" => Value::Bool(false), + _ => Value::String(s), + }, + (Some("array"), Value::String(s)) => match serde_json::from_str::(s.trim()) { + Ok(Value::Array(items)) => coerce_items(items, schema), + Ok(other) => Value::Array(vec![other]), + Err(_) => Value::Array(vec![Value::String(s)]), + }, + (Some("array"), Value::Array(items)) => coerce_items(items, schema), + (Some("array"), scalar @ (Value::Number(_) | Value::Bool(_))) => Value::Array(vec![scalar]), + (Some("object"), Value::String(s)) => match serde_json::from_str::(s.trim()) { + Ok(object @ Value::Object(_)) => coerce_to_schema(object, schema), + _ => Value::String(s), + }, + (Some("object"), object @ Value::Object(_)) => coerce_to_schema(object, schema), + (Some("string"), Value::Number(n)) => Value::String(n.to_string()), + (Some("null"), Value::String(s)) if s.trim() == "null" => Value::Null, + (_, value) => value, + } +} + +fn coerce_items(items: Vec, schema: &Value) -> Value { + let Some(item_schema) = schema.get("items") else { + return Value::Array(items); + }; + Value::Array( + items + .into_iter() + .map(|item| { + if let Value::String(s) = &item + && let Ok(decoded) = serde_json::from_str::(s) + && schema_type(item_schema).is_some_and(|t| t == "object" || t == "array") + { + return coerce_value(decoded, item_schema); + } + coerce_value(item, item_schema) + }) + .collect(), + ) +} diff --git a/crates/tinytools-agent/src/repair/json.rs b/crates/tinytools-agent/src/repair/json.rs new file mode 100644 index 0000000..736aa2a --- /dev/null +++ b/crates/tinytools-agent/src/repair/json.rs @@ -0,0 +1,531 @@ +//! Recovering a JSON **object** from the relaxed, damaged, or noise-wrapped +//! text a model emits where strict JSON was asked for. +//! +//! Every stage here is a shape a real model produced for tool arguments: +//! +//! * leaked chat-template markers glued to the value — `{"a":1}` +//! (OpenAI-compatible gateways that fail to strip the template); +//! * the model's string-delimiter token emitted as text — `[<|">x<|">]` +//! (Kimi-family via some gateways); +//! * a surrounding markdown fence; +//! * raw control characters inside strings (llama.cpp, tabs and newlines); +//! * trailing commas, and objects cut off before their closing brace +//! (streams that ended mid-call); +//! * redundant outer braces `{{…}}` the model piles on each time the previous +//! attempt bounced, and unquoted keys `{tool:"x"}`; +//! * typographic quotes `“…”`. +//! +//! The ladder is applied **only after strict parsing has failed**, and the +//! result is accepted only when it parses strictly *and* is an object, so a +//! scalar scraped out of noise can never masquerade as arguments. Stages are +//! cumulative and cheap: each one is a pass over the string, and most inputs +//! exit at the first or second rung. + +use serde_json::Value; + +/// Chat-template tool-call delimiters that gateways sometimes fail to strip +/// before placing a call in `function.arguments`, or that a model narrating a +/// call leaves glued to the JSON. Removed outright — they are structure, not +/// content. +pub const TEMPLATE_MARKERS: &[&str] = &[ + "<|tool_calls_section_begin|>", + "<|tool_calls_section_end|>", + "<|tool_call_argument_begin|>", + "<|tool_call_begin|>", + "<|tool_call_end|>", + "<|tool_call|>", + "<|tool_sep|>", + "", + "", + "", +]; + +/// Chat-template string-delimiter tokens emitted as literal text in place of +/// a `"` (Kimi-family models via GMI: `[<|">discord<|">]`). Substituted to +/// `"`, not deleted. Longer forms first so a substitution never leaves a +/// partial token behind. +pub const LEAKED_QUOTE_TOKENS: &[&str] = &["<|\"|>", "<|\">"]; + +/// Maximum redundant outer brace layers to peel. Bounds work on adversarial +/// `{{{{…}}}}` blobs while comfortably covering every depth seen in the wild. +const MAX_BRACE_PEEL: usize = 16; + +/// Maximum excess closing brackets trimmed from the tail. +const MAX_EXCESS_CLOSERS: usize = 50; + +/// Attempts to recover a strict-JSON **object** from relaxed or damaged text. +/// +/// Returns `None` when no conservative repair yields an object. Call this only +/// after `serde_json::from_str` has already failed on `raw`; a well-formed +/// object is returned unchanged by the first rung anyway, but the ladder is not +/// free. +#[must_use] +pub fn recover_object(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if let Some(object) = parse_object(trimmed) { + return Some(object); + } + + // Rung 1: unwrap and de-noise, then retry strictly. + let mut candidate = strip_code_fence(trimmed).to_string(); + candidate = strip_template_markers(&candidate); + candidate = normalize_leaked_quote_tokens(&candidate); + if let Some(object) = parse_object(candidate.trim()) { + return Some(object); + } + + // Rung 2: character-level damage a strict parser rejects outright. + candidate = escape_control_characters(&candidate); + candidate = strip_trailing_commas(&candidate); + if let Some(object) = parse_object(candidate.trim()) { + return Some(object); + } + if let Some(object) = parse_object(&balance_closers(candidate.trim())) { + return Some(object); + } + + // Rung 3: structural relaxations, retried at each brace depth. + if let Some(object) = peel_and_quote(candidate.trim()) { + return Some(object); + } + + // Rung 4: typographic quotes, then the whole ladder once more. + let straightened = straighten_quotes(&candidate); + if straightened != candidate { + if let Some(object) = parse_object(straightened.trim()) { + return Some(object); + } + if let Some(object) = peel_and_quote(&balance_closers(straightened.trim())) { + return Some(object); + } + } + + // Rung 5: a valid leading object followed by trailing noise. + leading_object(candidate.trim()) +} + +/// Parses `s` strictly and keeps it only when it is an object. +fn parse_object(s: &str) -> Option { + match serde_json::from_str::(s) { + Ok(value @ Value::Object(_)) => Some(value), + _ => None, + } +} + +/// The first complete JSON object at the front of `s`, ignoring what follows. +fn leading_object(s: &str) -> Option { + let mut values = serde_json::Deserializer::from_str(s).into_iter::(); + match values.next() { + Some(Ok(value @ Value::Object(_))) => Some(value), + _ => None, + } +} + +/// Alternates brace peeling and bare-key quoting until one parses. +fn peel_and_quote(s: &str) -> Option { + let mut layer = s.to_string(); + for _ in 0..=MAX_BRACE_PEEL { + if let Some(object) = parse_object(&layer) { + return Some(object); + } + let quoted = quote_bare_keys(&layer); + if quoted != layer + && let Some(object) = parse_object("ed) + { + return Some(object); + } + match peel_redundant_brace(&layer) { + Some(inner) => layer = inner, + None => break, + } + } + None +} + +/// Strips one surrounding markdown code fence, with or without a language tag. +#[must_use] +pub fn strip_code_fence(raw: &str) -> &str { + let trimmed = raw.trim(); + let Some(after_open) = trimmed.strip_prefix("```") else { + return trimmed; + }; + let body = match after_open.find('\n') { + Some(newline) + if after_open[..newline] + .trim() + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') => + { + &after_open[newline + 1..] + } + Some(_) => after_open, + None => return trimmed, + }; + body.trim_end() + .strip_suffix("```") + .map_or(trimmed, str::trim) +} + +/// Removes every [`TEMPLATE_MARKERS`] occurrence. +#[must_use] +pub fn strip_template_markers(raw: &str) -> String { + let mut out = raw.to_string(); + for marker in TEMPLATE_MARKERS { + if out.contains(marker) { + out = out.replace(marker, ""); + } + } + out +} + +/// Substitutes every [`LEAKED_QUOTE_TOKENS`] occurrence with `"`. +#[must_use] +pub fn normalize_leaked_quote_tokens(raw: &str) -> String { + let mut out = raw.to_string(); + for token in LEAKED_QUOTE_TOKENS { + if out.contains(token) { + out = out.replace(token, "\""); + } + } + out +} + +/// Escapes raw control characters (U+0000..U+001F) that appear *inside* JSON +/// string literals, which strict parsers reject. Characters outside strings +/// are left alone. +#[must_use] +pub fn escape_control_characters(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_string = false; + let mut escaped = false; + for ch in s.chars() { + if in_string { + if escaped { + escaped = false; + out.push(ch); + continue; + } + match ch { + '\\' => { + escaped = true; + out.push(ch); + } + '"' => { + in_string = false; + out.push(ch); + } + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if (c as u32) < 0x20 => { + use std::fmt::Write as _; + let _ = write!(out, "\\u{:04x}", c as u32); + } + c => out.push(c), + } + } else { + if ch == '"' { + in_string = true; + } + out.push(ch); + } + } + out +} + +/// Removes a `,` that directly precedes a `}` or `]` (whitespace allowed), +/// outside string literals. +#[must_use] +pub fn strip_trailing_commas(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut in_string = false; + let mut escaped = false; + let chars: Vec = s.chars().collect(); + let mut i = 0; + while i < chars.len() { + let ch = chars[i]; + if in_string { + out.push(ch); + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + i += 1; + continue; + } + if ch == '"' { + in_string = true; + out.push(ch); + i += 1; + continue; + } + if ch == ',' { + let mut j = i + 1; + while j < chars.len() && chars[j].is_whitespace() { + j += 1; + } + if j < chars.len() && (chars[j] == '}' || chars[j] == ']') { + i += 1; + continue; + } + } + out.push(ch); + i += 1; + } + out +} + +/// Appends the closers an unterminated value is missing, or trims a bounded +/// run of excess closers, so a call cut off mid-stream still parses. +#[must_use] +pub fn balance_closers(s: &str) -> String { + let mut stack: Vec = Vec::new(); + let mut in_string = false; + let mut escaped = false; + let mut excess = 0usize; + for ch in s.chars() { + if in_string { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + match ch { + '"' => in_string = true, + '{' => stack.push('}'), + '[' => stack.push(']'), + '}' | ']' => { + if stack.last() == Some(&ch) { + stack.pop(); + } else { + excess += 1; + } + } + _ => {} + } + } + if in_string { + // A value cut off mid-string cannot be completed honestly: guessing + // where it ended would hand a tool a silently truncated argument. + return s.to_string(); + } + let mut out = s.to_string(); + if excess > 0 && excess <= MAX_EXCESS_CLOSERS && stack.is_empty() { + let mut trimmed = out.trim_end().to_string(); + for _ in 0..excess { + if trimmed.ends_with('}') || trimmed.ends_with(']') { + trimmed.pop(); + trimmed = trimmed.trim_end().to_string(); + } + } + return trimmed; + } + while let Some(closer) = stack.pop() { + out.push(closer); + } + out +} + +/// Replaces typographic double and single quotes with their ASCII forms. +#[must_use] +pub fn straighten_quotes(s: &str) -> String { + s.chars() + .map(|c| match c { + '\u{201C}' | '\u{201D}' | '\u{201E}' | '\u{201F}' => '"', + '\u{2018}' | '\u{2019}' => '\'', + other => other, + }) + .collect() +} + +/// Peels one redundant outer brace layer wrapping exactly one object. +fn peel_redundant_brace(s: &str) -> Option { + let trimmed = s.trim(); + let inner = trimmed.strip_prefix('{')?.strip_suffix('}')?.trim(); + if inner.starts_with('{') && object_spans_all(inner) { + Some(inner.to_string()) + } else { + None + } +} + +/// Whether the object opened at byte 0 closes exactly at the end of `s`. +fn object_spans_all(s: &str) -> bool { + if !s.starts_with('{') { + return false; + } + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for (idx, ch) in s.char_indices() { + if in_string { + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + match ch { + '"' => in_string = true, + '{' => depth += 1, + '}' => { + depth = match depth.checked_sub(1) { + Some(d) => d, + None => return false, + }; + if depth == 0 { + return idx + ch.len_utf8() == s.len(); + } + } + _ => {} + } + } + false +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Container { + Object, + Array, +} + +/// Reads a key opened by `"` or `'` up to a closing quote that is followed by +/// `:`. Returns the key text and the bytes consumed. `None` when the quoting +/// is already correct (so the caller leaves it alone) or when the run does +/// not look like a key at all. +fn take_quoted_key(rest: &str) -> Option<(String, usize)> { + let mut chars = rest.char_indices(); + let (_, open) = chars.next()?; + let mut key = String::new(); + for (idx, ch) in chars { + match ch { + '"' | '\'' => { + let after = &rest[idx + ch.len_utf8()..]; + if after.trim_start().starts_with(':') { + if open == '"' && ch == '"' { + return None; + } + return Some((key, idx + ch.len_utf8())); + } + key.push(ch); + } + '\n' | '\r' | '{' | '}' | '[' | ']' | ':' => return None, + _ => key.push(ch), + } + } + None +} + +/// Quotes bare and single-quoted object **keys**, string- and array-aware so +/// values and array literals are never rewritten. +#[must_use] +pub fn quote_bare_keys(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 8); + let mut stack: Vec = Vec::new(); + let mut expect_key = false; + let mut in_string = false; + let mut escaped = false; + let mut chars = s.char_indices().peekable(); + + while let Some((idx, ch)) = chars.next() { + if in_string { + out.push(ch); + if escaped { + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + in_string = false; + } + continue; + } + + match ch { + '"' | '\'' if expect_key && matches!(stack.last(), Some(Container::Object)) => { + if let Some((key, consumed)) = take_quoted_key(&s[idx..]) { + out.push('"'); + out.push_str(&key.replace('\\', r"\\").replace('"', "\\\"")); + out.push('"'); + while chars.peek().is_some_and(|&(next, _)| next < idx + consumed) { + chars.next(); + } + expect_key = false; + } else { + in_string = true; + expect_key = false; + out.push(ch); + } + } + '"' => { + in_string = true; + expect_key = false; + out.push(ch); + } + '{' => { + stack.push(Container::Object); + expect_key = true; + out.push(ch); + } + '}' | ']' => { + stack.pop(); + expect_key = false; + out.push(ch); + } + '[' => { + stack.push(Container::Array); + expect_key = false; + out.push(ch); + } + ',' => { + expect_key = matches!(stack.last(), Some(Container::Object)); + out.push(ch); + } + ':' => { + expect_key = false; + out.push(ch); + } + c if c.is_whitespace() => out.push(ch), + c if expect_key + && matches!(stack.last(), Some(Container::Object)) + && (c.is_ascii_alphabetic() || c == '_') => + { + let start = idx; + let mut end = idx + c.len_utf8(); + while let Some(&(next_idx, next_ch)) = chars.peek() { + if next_ch.is_ascii_alphanumeric() + || next_ch == '_' + || next_ch == '-' + || next_ch == '.' + { + end = next_idx + next_ch.len_utf8(); + chars.next(); + } else { + break; + } + } + out.push('"'); + out.push_str(&s[start..end]); + out.push('"'); + expect_key = false; + } + _ => { + expect_key = false; + out.push(ch); + } + } + } + out +} diff --git a/crates/tinytools-agent/src/repair/mod.rs b/crates/tinytools-agent/src/repair/mod.rs new file mode 100644 index 0000000..764e88e --- /dev/null +++ b/crates/tinytools-agent/src/repair/mod.rs @@ -0,0 +1,21 @@ +//! Repairs applied to what a model wrote *after* a call has been located. +//! +//! Locating a call and reading it are separate problems. The grammars in +//! [`crate::parse`] answer "where is the call and what did the model write +//! there"; this module answers "what did it mean" when the answer is not +//! strict JSON, not the exact tool name, or not the argument shape the schema +//! declares. Keeping the two apart is what lets every grammar share one repair +//! path, so a fix for a Kimi quote sentinel helps a DSML block equally. +//! +//! Three concerns, three submodules: +//! +//! * [`json`] — a JSON object from relaxed or damaged text; +//! * [`name`] — the offered tool a damaged name refers to; +//! * [`args`] — the argument object's shape against its schema. + +pub mod args; +pub mod json; +pub mod name; + +#[cfg(test)] +mod test; diff --git a/crates/tinytools-agent/src/repair/name.rs b/crates/tinytools-agent/src/repair/name.rs new file mode 100644 index 0000000..87bfc51 --- /dev/null +++ b/crates/tinytools-agent/src/repair/name.rs @@ -0,0 +1,231 @@ +//! Resolving the tool name a model wrote to the tool it meant. +//! +//! Small models damage names in a handful of recurring ways, all observed on +//! live hosts: XML attributes leaked into the name (`terminal" parameter="command"`), +//! a namespace prefix the model saw in a chat template (`functions.read`, +//! `tools/read`), a stray suffix (`TodoTool_tool`), the wrong case or +//! separator (`Write File`, `write-file`), or one typo (`serach`). +//! +//! Resolution is conservative by construction. Nothing is rewritten unless the +//! result is a **unique** offered tool, and nothing is invented: with no +//! known-tool set the only change is dropping trailing junk after a quote or +//! angle bracket, which can never *create* a match. That keeps the property +//! the parser depends on — a plain JSON answer that happens to carry a `name` +//! is not turned into a call by fuzzy matching. + +/// How a name was resolved. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NameResolution { + /// The name to dispatch. + pub name: String, + /// Whether it differs from what the model wrote. + pub repaired: bool, + /// Whether it matches an offered tool. Always `false` when no tools were + /// supplied. + pub known: bool, +} + +/// Namespace prefixes a chat template may have taught the model. +const NAMESPACE_PREFIXES: &[&str] = &["functions.", "functions/", "tools.", "tools/", "tool."]; + +/// Suffixes a model appends when it confuses the tool with its class name. +const TOOL_SUFFIXES: &[&str] = &["_tool", "-tool", "Tool", "tool"]; + +/// Maximum edit distance accepted for a fuzzy match. +const MAX_EDIT_DISTANCE: usize = 2; + +/// Resolves `raw` against `known` tools. +/// +/// The steps run in order and stop at the first that yields a known tool: +/// exact match, junk trimmed, namespace prefix dropped, separators and case +/// normalised, class suffix dropped, then a unique edit-distance match. A raw +/// name that resolves to nothing is returned trimmed of junk so the host's +/// unknown-tool policy sees something sensible. +#[must_use] +pub fn resolve(raw: &str, known: &[String]) -> NameResolution { + let original = raw.trim(); + let trimmed = trim_junk(original); + + if known.is_empty() { + return NameResolution { + repaired: trimmed != original, + name: trimmed.to_string(), + known: false, + }; + } + + let found = |candidate: &str| known.iter().any(|k| k == candidate); + + if found(original) { + return resolved(original, original); + } + if found(trimmed) { + return resolved(original, trimmed); + } + + let unprefixed = strip_namespace(trimmed); + if found(unprefixed) { + return resolved(original, unprefixed); + } + + let normalized = normalize(unprefixed); + if let Some(hit) = unique_by_normalized(&normalized, known) { + return resolved(original, hit); + } + + // Class-name suffixes, stripped up to twice (`TodoTool_tool`). + let mut stem = unprefixed.to_string(); + for _ in 0..2 { + let Some(shorter) = TOOL_SUFFIXES + .iter() + .find_map(|suffix| stem.strip_suffix(suffix)) + .map(str::to_string) + else { + break; + }; + stem = shorter; + let stem_norm = normalize(&stem); + if let Some(hit) = unique_by_normalized(&stem_norm, known) { + return resolved(original, hit); + } + } + + if let Some(hit) = unique_fuzzy(&normalized, known) { + return resolved(original, hit); + } + + NameResolution { + repaired: trimmed != original, + name: trimmed.to_string(), + known: false, + } +} + +fn resolved(original: &str, name: &str) -> NameResolution { + NameResolution { + name: name.to_string(), + repaired: name != original, + known: true, + } +} + +/// Cuts the name at the first character that cannot be part of one. +fn trim_junk(s: &str) -> &str { + let end = s + .find(|c: char| { + matches!( + c, + '"' | '\'' | '<' | '>' | '(' | ')' | '\n' | '\r' | ':' | '=' | '{' | '[' + ) + }) + .unwrap_or(s.len()); + s[..end].trim() +} + +fn strip_namespace(s: &str) -> &str { + for prefix in NAMESPACE_PREFIXES { + if let Some(rest) = s.strip_prefix(prefix) + && !rest.is_empty() + { + return rest; + } + } + s +} + +/// Lower-case, `snake_case`, separators unified: `Write File` / `write-file` / +/// `WriteFile` all become `write_file`. +fn normalize(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 4); + let mut prev_lower = false; + for ch in s.chars() { + match ch { + '-' | ' ' | '.' | '/' => { + if !out.ends_with('_') { + out.push('_'); + } + prev_lower = false; + } + c if c.is_uppercase() => { + if prev_lower && !out.ends_with('_') { + out.push('_'); + } + out.extend(c.to_lowercase()); + prev_lower = false; + } + c => { + out.push(c); + prev_lower = c.is_lowercase() || c.is_ascii_digit(); + } + } + } + out.trim_matches('_').to_string() +} + +/// The single known tool whose normalized form equals `normalized`, or +/// `None` when zero or several tie — normalization collapses distinct raw +/// names (`read_file` and `read-file` both become `read_file`), so a +/// non-unique hit must not silently pick whichever offered tool sorts +/// first. +fn unique_by_normalized<'a>(normalized: &str, known: &'a [String]) -> Option<&'a str> { + let mut hit: Option<&str> = None; + for candidate in known { + if normalize(candidate) == normalized { + if hit.is_some() { + return None; + } + hit = Some(candidate); + } + } + hit +} + +/// The single known tool within [`MAX_EDIT_DISTANCE`] of `needle`, or `None` +/// when there are zero or several — an ambiguous match must not dispatch. +fn unique_fuzzy<'a>(needle: &str, known: &'a [String]) -> Option<&'a str> { + if needle.len() < 4 { + return None; + } + let budget = MAX_EDIT_DISTANCE.min(needle.len() / 3); + if budget == 0 { + return None; + } + let mut best: Option<(&str, usize)> = None; + let mut ambiguous = false; + for candidate in known { + let distance = edit_distance(needle, &normalize(candidate)); + if distance > budget { + continue; + } + match best { + Some((_, d)) if d == distance => ambiguous = true, + Some((_, d)) if d < distance => {} + _ => { + best = Some((candidate, distance)); + ambiguous = false; + } + } + } + if ambiguous { + None + } else { + best.map(|(name, _)| name) + } +} + +/// Levenshtein distance over chars. +fn edit_distance(a: &str, b: &str) -> usize { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + let mut prev: Vec = (0..=b.len()).collect(); + let mut cur = vec![0; b.len() + 1]; + for (i, ca) in a.iter().enumerate() { + cur[0] = i + 1; + for (j, cb) in b.iter().enumerate() { + let cost = usize::from(ca != cb); + cur[j + 1] = (prev[j + 1] + 1).min(cur[j] + 1).min(prev[j] + cost); + } + std::mem::swap(&mut prev, &mut cur); + } + prev[b.len()] +} diff --git a/crates/tinytools-agent/src/repair/test/args.rs b/crates/tinytools-agent/src/repair/test/args.rs new file mode 100644 index 0000000..6be2f30 --- /dev/null +++ b/crates/tinytools-agent/src/repair/test/args.rs @@ -0,0 +1,233 @@ +//! Argument shape repair. + +use crate::repair::args::{ + accepts_object, coerce_to_schema, decode, from_call_object, unwrap_envelope, +}; +use serde_json::json; + +fn city_schema() -> serde_json::Value { + json!({ "type": "object", "properties": { "city": { "type": "string" } }, "required": ["city"] }) +} + +fn valid_city(value: &serde_json::Value) -> bool { + value.get("city").is_some_and(serde_json::Value::is_string) +} + +#[test] +fn decode_handles_strings_fences_and_relaxed_json() { + assert_eq!(decode(Some(&json!("{\"a\":1}"))), json!({ "a": 1 })); + assert_eq!( + decode(Some(&json!("```json\n{\"a\":1}\n```"))), + json!({ "a": 1 }) + ); + assert_eq!(decode(Some(&json!("{a:1}"))), json!({ "a": 1 })); + assert_eq!(decode(Some(&json!("garbage"))), json!({})); + assert_eq!(decode(None), json!({})); +} + +#[test] +fn argument_key_aliases_are_read_in_priority_order() { + assert_eq!( + from_call_object(&json!({ "args": { "x": 1 } })), + json!({ "x": 1 }) + ); + assert_eq!( + from_call_object(&json!({ "arguments": { "x": 1 }, "input": { "x": 2 } })), + json!({ "x": 1 }) + ); +} + +#[test] +fn envelopes_are_unwrapped_only_when_the_inner_value_validates() { + let schema = city_schema(); + for wrapped in [ + json!({ "type": "object", "required": ["city"], "properties": { "city": "Paris" } }), + json!({ "properties": {}, "required": [], "arguments": { "city": "Paris" } }), + json!({ "param": { "city": "Paris" } }), + ] { + assert_eq!( + unwrap_envelope(&wrapped, &schema, &valid_city), + Some(json!({ "city": "Paris" })), + "{wrapped}" + ); + } + assert_eq!( + unwrap_envelope( + &json!({ "param": { "town": "Paris" } }), + &schema, + &valid_city + ), + None + ); +} + +#[test] +fn a_declared_parameter_is_never_treated_as_an_envelope() { + let schema = json!({ "type": "object", "properties": { "input": { "type": "object" } } }); + let arguments = json!({ "input": { "city": "Paris" } }); + assert_eq!(unwrap_envelope(&arguments, &schema, &valid_city), None); +} + +#[test] +fn accepts_object_reads_every_schema_spelling() { + assert!(accepts_object(&json!({ "type": "object" }))); + assert!(accepts_object(&json!({ "type": ["object", "null"] }))); + assert!(accepts_object(&json!({ "properties": {} }))); + assert!(!accepts_object(&json!({ "type": "string" }))); +} + +#[test] +fn scalars_are_coerced_to_the_declared_type() { + let schema = json!({ + "type": "object", + "properties": { + "n": { "type": "integer" }, + "f": { "type": "number" }, + "b": { "type": "boolean" }, + "list": { "type": "array", "items": { "type": "string" } }, + "nested": { "type": "object", "properties": { "k": { "type": "integer" } } }, + "s": { "type": "string" } + } + }); + let out = coerce_to_schema( + json!({ "n": "42", "f": "3.5", "b": "true", "list": "[\"a\",\"b\"]", "nested": "{\"k\":\"7\"}", "s": 5, "extra": "x" }), + &schema, + ); + assert_eq!( + out, + json!({ "n": 42, "f": 3.5, "b": true, "list": ["a", "b"], "nested": { "k": 7 }, "s": "5", "extra": "x" }) + ); +} + +#[test] +fn coerce_to_schema_leaves_a_non_object_value_untouched() { + let schema = json!({ "type": "object", "properties": { "n": { "type": "integer" } } }); + assert_eq!( + coerce_to_schema(json!(["not", "an", "object"]), &schema), + json!(["not", "an", "object"]) + ); +} + +#[test] +fn coerce_to_schema_with_no_declared_properties_passes_the_object_through() { + let schema = json!({ "type": "object" }); + let arguments = json!({ "x": 1, "y": "z" }); + assert_eq!(coerce_to_schema(arguments.clone(), &schema), arguments); +} + +#[test] +fn a_nullable_type_array_still_drives_scalar_coercion() { + let schema = json!({ + "type": "object", + "properties": { "n": { "type": ["null", "integer"] } } + }); + assert_eq!( + coerce_to_schema(json!({ "n": "5" }), &schema), + json!({ "n": 5 }) + ); +} + +#[test] +fn boolean_false_spellings_are_coerced() { + let schema = json!({ "type": "object", "properties": { "b": { "type": "boolean" } } }); + for spelling in ["false", "False", "FALSE"] { + assert_eq!( + coerce_to_schema(json!({ "b": spelling }), &schema), + json!({ "b": false }), + "{spelling}" + ); + } +} + +#[test] +fn an_array_typed_string_that_decodes_to_a_scalar_is_wrapped() { + let schema = json!({ "type": "object", "properties": { "list": { "type": "array" } } }); + assert_eq!( + coerce_to_schema(json!({ "list": "5" }), &schema), + json!({ "list": [5] }) + ); +} + +#[test] +fn a_native_array_value_is_coerced_by_item_schema() { + let schema = json!({ + "type": "object", + "properties": { "list": { "type": "array", "items": { "type": "integer" } } } + }); + assert_eq!( + coerce_to_schema(json!({ "list": ["1", "2"] }), &schema), + json!({ "list": [1, 2] }) + ); +} + +#[test] +fn an_object_typed_string_that_fails_to_decode_is_left_as_a_string() { + let schema = json!({ + "type": "object", + "properties": { "nested": { "type": "object" } } + }); + assert_eq!( + coerce_to_schema(json!({ "nested": "not json" }), &schema), + json!({ "nested": "not json" }) + ); +} + +#[test] +fn a_native_object_value_is_coerced_by_its_nested_schema() { + let schema = json!({ + "type": "object", + "properties": { + "nested": { + "type": "object", + "properties": { "k": { "type": "integer" } } + } + } + }); + assert_eq!( + coerce_to_schema(json!({ "nested": { "k": "7" } }), &schema), + json!({ "nested": { "k": 7 } }) + ); +} + +#[test] +fn an_array_without_an_items_schema_is_left_unchanged() { + let schema = json!({ "type": "object", "properties": { "list": { "type": "array" } } }); + assert_eq!( + coerce_to_schema(json!({ "list": ["a", 1, true] }), &schema), + json!({ "list": ["a", 1, true] }) + ); +} + +#[test] +fn array_items_that_are_json_encoded_strings_are_decoded_per_item_schema() { + let schema = json!({ + "type": "object", + "properties": { + "list": { + "type": "array", + "items": { "type": "object", "properties": { "k": { "type": "integer" } } } + } + } + }); + assert_eq!( + coerce_to_schema(json!({ "list": ["{\"k\":\"7\"}"] }), &schema), + json!({ "list": [{ "k": 7 }] }) + ); +} + +#[test] +fn unconvertible_scalars_are_left_for_the_validator() { + let schema = json!({ "type": "object", "properties": { "n": { "type": "integer" }, "l": { "type": "array" } } }); + assert_eq!( + coerce_to_schema(json!({ "n": "many" }), &schema), + json!({ "n": "many" }) + ); + assert_eq!( + coerce_to_schema(json!({ "l": "solo" }), &schema), + json!({ "l": ["solo"] }) + ); + assert_eq!( + coerce_to_schema(json!({ "l": 3 }), &schema), + json!({ "l": [3] }) + ); +} diff --git a/crates/tinytools-agent/src/repair/test/json.rs b/crates/tinytools-agent/src/repair/test/json.rs new file mode 100644 index 0000000..d15ff59 --- /dev/null +++ b/crates/tinytools-agent/src/repair/test/json.rs @@ -0,0 +1,156 @@ +//! JSON repair ladder. + +use crate::repair::json::{ + balance_closers, escape_control_characters, recover_object, strip_code_fence, + strip_trailing_commas, +}; +use serde_json::json; + +#[test] +fn strict_objects_pass_through() { + assert_eq!(recover_object(r#"{"a":1}"#), Some(json!({"a": 1}))); + assert_eq!(recover_object(""), None); + assert_eq!( + recover_object("[1,2]"), + None, + "a non-object is never an object" + ); +} + +#[test] +fn repairs_single_quoted_and_mismatched_keys() { + assert_eq!( + recover_object(r#"{"name":"get_weather","parameters':{'city':"Paris"}}"#), + Some(json!({ "name": "get_weather", "parameters": { "city": "Paris" } })) + ); + assert_eq!( + recover_object(r#"{'city':"Paris"}"#), + Some(json!({ "city": "Paris" })) + ); +} + +#[test] +fn single_quoted_values_are_left_unrepaired() { + // An apostrophe in a value is ordinary English; never rewrite it. + assert_eq!(recover_object(r"{'city':'Paris'}"), None); +} + +#[test] +fn an_apostrophe_inside_a_well_formed_key_is_not_a_delimiter() { + assert_eq!( + recover_object(r#"{"it's fine":1,bare:2}"#), + Some(json!({ "it's fine": 1, "bare": 2 })) + ); +} + +#[test] +fn quotes_unquoted_keys_and_leaves_literals() { + assert_eq!( + recover_object(r#"{toolkits:["discord"]}"#), + Some(json!({ "toolkits": ["discord"] })) + ); + assert_eq!( + recover_object(r#"{include_unconnected:true,toolkits:["discord"],n:null}"#), + Some(json!({ "include_unconnected": true, "toolkits": ["discord"], "n": null })) + ); + assert_eq!( + recover_object(r#"{query:"from:john,to:x",n:1}"#), + Some(json!({ "query": "from:john,to:x", "n": 1 })) + ); +} + +#[test] +fn substitutes_leaked_quote_tokens() { + assert_eq!( + recover_object(r#"{toolkits:[<|">discord<|">]}"#), + Some(json!({ "toolkits": ["discord"] })) + ); + assert_eq!( + recover_object(r#"{label_ids:[<|"|>INBOX<|"|>]}"#), + Some(json!({ "label_ids": ["INBOX"] })) + ); +} + +#[test] +fn peels_redundant_brace_layers() { + assert_eq!(recover_object(r#"{{{"a":1}}}"#), Some(json!({ "a": 1 }))); + assert_eq!( + recover_object(r#"{{tool:"X",arguments:{guild_id:"Y"}}}"#), + Some(json!({ "tool": "X", "arguments": { "guild_id": "Y" } })) + ); +} + +#[test] +fn strips_leaked_template_markers() { + assert_eq!( + recover_object(r#"{"a":1}"#), + Some(json!({ "a": 1 })) + ); + assert_eq!( + recover_object(r#"{"a":1}<|tool_calls_section_end|>"#), + Some(json!({ "a": 1 })) + ); + assert_eq!( + recover_object(r#"{"a":1}{"b":2}"#), + Some(json!({ "a": 1 })) + ); +} + +#[test] +fn strips_a_code_fence() { + assert_eq!( + recover_object("```json\n{\"a\":1}\n```"), + Some(json!({ "a": 1 })) + ); + assert_eq!(strip_code_fence("```\n{}\n```"), "{}"); + assert_eq!(strip_code_fence("plain"), "plain"); +} + +#[test] +fn trailing_commas_and_missing_closers_are_repaired() { + assert_eq!( + recover_object(r#"{"a": [1, 2,]}"#), + Some(json!({ "a": [1, 2] })) + ); + assert_eq!( + recover_object(r#"{"command": "ls -la", "timeout": 30"#), + Some(json!({ "command": "ls -la", "timeout": 30 })) + ); + assert_eq!( + recover_object(r#"{"a":{"b":1}}}}"#), + Some(json!({ "a": { "b": 1 } })) + ); + assert_eq!( + strip_trailing_commas(r#"{"a":"x,}","b":1,}"#), + r#"{"a":"x,}","b":1}"# + ); + assert_eq!(balance_closers(r#"{"a":[1"#), r#"{"a":[1]}"#); +} + +#[test] +fn control_characters_inside_strings_are_escaped() { + let raw = "{\"cmd\":\"ls\n-la\t\"}"; + assert_eq!(recover_object(raw), Some(json!({ "cmd": "ls\n-la\t" }))); + assert_eq!( + escape_control_characters("{\"a\":\"x\ny\"}"), + "{\"a\":\"x\\ny\"}" + ); +} + +#[test] +fn typographic_quotes_are_straightened_last() { + assert_eq!( + recover_object("{“path”: “a.txt”}"), + Some(json!({ "path": "a.txt" })) + ); +} + +#[test] +fn truly_unrecoverable_input_is_none() { + assert_eq!(recover_object("not json at all"), None); + assert_eq!( + recover_object(r#"{"truncated": "val"#), + None, + "an open string cannot be guessed" + ); +} diff --git a/crates/tinytools-agent/src/repair/test/mod.rs b/crates/tinytools-agent/src/repair/test/mod.rs new file mode 100644 index 0000000..c71ae28 --- /dev/null +++ b/crates/tinytools-agent/src/repair/test/mod.rs @@ -0,0 +1,6 @@ +//! Unit tests for the repair ladders. +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +mod args; +mod json; +mod name; diff --git a/crates/tinytools-agent/src/repair/test/name.rs b/crates/tinytools-agent/src/repair/test/name.rs new file mode 100644 index 0000000..e0a456a --- /dev/null +++ b/crates/tinytools-agent/src/repair/test/name.rs @@ -0,0 +1,97 @@ +//! Tool-name resolution. + +use crate::repair::name::resolve; + +fn known() -> Vec { + ["terminal", "read_file", "write_file", "todo", "search_web"] + .iter() + .map(ToString::to_string) + .collect() +} + +#[test] +fn exact_names_are_untouched() { + let r = resolve("read_file", &known()); + assert_eq!(r.name, "read_file"); + assert!(!r.repaired); + assert!(r.known); +} + +#[test] +fn leaked_xml_attributes_are_trimmed() { + assert_eq!( + resolve("terminal\" parameter=\"command\" string=\"true", &known()).name, + "terminal" + ); + assert_eq!(resolve("terminal\"", &known()).name, "terminal"); + assert_eq!(resolve("read_file(", &known()).name, "read_file"); +} + +#[test] +fn namespace_prefixes_are_dropped() { + assert_eq!(resolve("functions.read_file", &known()).name, "read_file"); + assert_eq!(resolve("tools/read_file", &known()).name, "read_file"); +} + +#[test] +fn case_and_separators_are_normalised() { + assert_eq!(resolve("Read File", &known()).name, "read_file"); + assert_eq!(resolve("read-file", &known()).name, "read_file"); + assert_eq!(resolve("ReadFile", &known()).name, "read_file"); + assert_eq!(resolve("TodoTool_tool", &known()).name, "todo"); +} + +#[test] +fn a_single_typo_resolves_when_unique() { + assert_eq!(resolve("raed_file", &known()).name, "read_file"); + assert_eq!(resolve("serach_web", &known()).name, "search_web"); +} + +#[test] +fn ambiguous_or_distant_names_are_not_invented() { + let r = resolve( + "xead_file", + &["read_file".to_string(), "bead_file".to_string()], + ); + assert!(!r.known, "two equally close candidates must not dispatch"); + let r = resolve("launch_missiles", &known()); + assert_eq!(r.name, "launch_missiles"); + assert!(!r.known); + assert!(!r.repaired); +} + +#[test] +fn a_normalized_collision_between_offered_tools_is_not_dispatched() { + // `read_file` and `read-file` both normalize to `read_file`. A damaged + // name that only matches after normalization must not be dispatched to + // whichever of the two colliding tools happens to be offered first. + let known: Vec = ["read_file", "read-file"] + .iter() + .map(ToString::to_string) + .collect(); + let r = resolve("Read File", &known); + assert!( + !r.known, + "an ambiguous normalized match must not dispatch: {r:?}" + ); + + // The class-suffix path re-normalizes after stripping `_tool`/`Tool` and + // must apply the same uniqueness rule: `to_do` and `ToDo` both + // normalize to `to_do`. + let known: Vec = ["to_do", "ToDo"].iter().map(ToString::to_string).collect(); + let r = resolve("ToDoTool", &known); + assert!( + !r.known, + "an ambiguous suffix-stripped match must not dispatch: {r:?}" + ); +} + +#[test] +fn without_known_tools_only_junk_is_trimmed() { + let r = resolve("terminal\" parameter", &[]); + assert_eq!(r.name, "terminal"); + assert!(r.repaired); + assert!(!r.known); + let r = resolve("Read File", &[]); + assert_eq!(r.name, "Read File"); +} diff --git a/crates/tinytools-agent/src/stream/mod.rs b/crates/tinytools-agent/src/stream/mod.rs new file mode 100644 index 0000000..5ae526e --- /dev/null +++ b/crates/tinytools-agent/src/stream/mod.rs @@ -0,0 +1,158 @@ +//! Stripping tool-call markup from a text stream *as fragments arrive*. +//! +//! A terminal-response parse only cleans the aggregated answer. A consumer +//! rendering live text deltas would still watch `{"name":…` stream +//! through character by character. This scrubber closes that gap: it emits +//! only the text that is provably not part of a tool-call block, holding back +//! any tail that could still become one — a partial opener such as `, + /// Diagnostics from blocks that completed. + pub diagnostics: Vec, +} + +/// Stateful, grammar-aware scrubber for streamed visible text. +#[derive(Debug, Default)] +pub struct StreamScrubber { + buf: String, + known_tools: Vec, + registry: Option>, +} + +impl StreamScrubber { + /// An empty scrubber with no known tools and no registry. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Sets the tools offered this turn, enabling name repair. + #[must_use] + pub fn with_known_tools(mut self, tools: Vec) -> Self { + self.known_tools = tools; + self + } + + /// Sets the P-Format registry. + #[must_use] + pub fn with_registry(mut self, registry: Arc) -> Self { + self.registry = Some(registry); + self + } + + fn options(&self) -> ParseOptions<'_> { + let mut options = ParseOptions::new() + .with_known_tools(&self.known_tools) + .without_bare_json(); + if let Some(registry) = self.registry.as_deref() { + options = options.with_registry(registry); + } + options + } + + /// Feeds the next fragment and returns what is safe to release now. + pub fn feed(&mut self, fragment: &str) -> StreamStep { + self.buf.push_str(fragment); + let options = self.options(); + let scan = scan(&self.buf, &options, ScanMode::Stream); + + let mut text = String::new(); + let consumed = if let Some(start) = scan.pending { + for range in &scan.kept { + text.push_str(&self.buf[range.clone()]); + } + start + } else { + let mut consumed = self.buf.len(); + let last = scan.kept.len().saturating_sub(1); + for (index, range) in scan.kept.iter().enumerate() { + if index == last { + let tail = &self.buf[range.clone()]; + let hold = hold_from(tail); + text.push_str(&tail[..hold]); + consumed = range.start + hold; + } else { + text.push_str(&self.buf[range.clone()]); + } + } + consumed + }; + let (calls, diagnostics) = resolve_names(scan.calls, scan.diagnostics, &options); + self.buf.drain(..consumed); + StreamStep { + text, + calls, + diagnostics, + } + } + + /// Drains the remainder once no more fragments will arrive. A complete + /// block still buffered yields its calls; a dangling opener is released + /// verbatim — with the stream ended it was never a call. + pub fn flush(&mut self) -> StreamStep { + let options = self.options(); + let scan = scan(&self.buf, &options, ScanMode::Batch); + let mut text = String::new(); + for range in &scan.kept { + text.push_str(&self.buf[range.clone()]); + } + let (calls, diagnostics) = resolve_names(scan.calls, scan.diagnostics, &options); + self.buf.clear(); + StreamStep { + text, + calls, + diagnostics, + } + } +} + +/// Byte index in `tail` from which the trailing bytes must be held because +/// they are a proper prefix of a block opener (` usize { + let len = tail.len(); + let mut best = len; + for opener in all_openers() { + let max = (opener.len() - 1).min(len); + for k in (1..=max).rev() { + if opener.is_char_boundary(k) + && tail.is_char_boundary(len - k) + && tail.as_bytes()[len - k..].eq_ignore_ascii_case(&opener.as_bytes()[..k]) + { + best = best.min(len - k); + break; + } + } + } + best +} + +#[cfg(test)] +mod test; diff --git a/crates/tinytools-agent/src/stream/test.rs b/crates/tinytools-agent/src/stream/test.rs new file mode 100644 index 0000000..874b7b8 --- /dev/null +++ b/crates/tinytools-agent/src/stream/test.rs @@ -0,0 +1,186 @@ +//! Unit tests for the stream scrubber. +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +use super::StreamScrubber; +use crate::parse::parse_tool_calls; + +/// Feeds every fragment and appends the flush, returning the visible text and +/// every call released along the way. +fn scrub_all(fragments: &[&str]) -> (String, usize) { + let mut s = StreamScrubber::new(); + let mut out = String::new(); + let mut calls = 0; + for f in fragments { + let step = s.feed(f); + out.push_str(&step.text); + calls += step.calls.len(); + } + let step = s.flush(); + out.push_str(&step.text); + calls += step.calls.len(); + (out, calls) +} + +#[test] +fn plain_text_passes_through_unchanged() { + assert_eq!( + scrub_all(&["hello ", "world", " done"]).0, + "hello world done" + ); +} + +#[test] +fn a_complete_block_in_one_fragment_is_dropped_and_its_call_released() { + let (out, calls) = + scrub_all(&[r#"before {"name":"x","arguments":{}} after"#]); + assert_eq!(out, "before after"); + assert_eq!(calls, 1); +} + +#[test] +fn markup_split_across_fragments_never_leaks() { + let (out, calls) = scrub_all(&[ + "answer: ", + "{\"name\":", + "\"x\",\"arguments\":{\"a\":1}}", + " end", + ]); + assert_eq!(out, "answer: end"); + assert_eq!(calls, 1); +} + +#[test] +fn a_partial_open_marker_is_held_not_emitted() { + let mut s = StreamScrubber::new(); + let first = s.feed("value {\"name\":\"x\",\"arguments\":{}}!"); + assert_eq!(second.text, "!"); + assert_eq!(second.calls.len(), 1); + assert_eq!(s.flush().text, ""); +} + +#[test] +fn an_attribute_open_form_split_mid_tag_is_held() { + let mut s = StreamScrubber::new(); + let mut out = String::new(); + out.push_str(&s.feed("ok {\"name\":\"x\",\"arguments\":{}}") + .text, + ); + out.push_str(&s.flush().text); + assert_eq!(out, "ok "); +} + +#[test] +fn deepseek_delimiters_split_across_fragments_are_scrubbed() { + let (out, calls) = scrub_all(&[ + "r <｜tool▁ca", + "ll▁begin｜>{\"name\":\"a\",\"arguments\":{}}<｜tool▁call", + "▁end｜> s", + ]); + assert_eq!(out, "r s"); + assert!(!out.contains("tool▁call")); + assert_eq!(calls, 1); +} + +#[test] +fn dsml_split_across_fragments_is_scrubbed() { + let (out, calls) = scrub_all(&[ + "Sure. <｜DS", + "ML｜tool_calls><｜DSML｜invoke name=\"read\">{\"path\":\"a\"}", + " Done.", + ]); + assert_eq!(out.trim(), "Sure. Done.".trim()); + assert!(!out.contains("DSML"), "{out}"); + assert_eq!(calls, 1); +} + +#[test] +fn plural_tool_calls_prose_is_not_held() { + assert_eq!( + scrub_all(&["the ", "key"]).0, + "the key" + ); +} + +#[test] +fn flush_surfaces_a_dangling_open_verbatim_untrimmed() { + let mut s = StreamScrubber::new(); + let mid = s.feed(" a commentary to=functions.read<|message|>{\"path\":"); + assert_eq!(first.text, ""); + let second = s.feed("\"a\"}<|call|>tail"); + assert_eq!(second.text, "tail"); + assert_eq!(second.calls[0].name, "read"); +} + +#[test] +fn a_fenced_example_split_across_fragments_never_leaks_a_call() { + // A language-tagged fence opener released before its closing fence + // arrives would erase the only record that the buffer is still inside + // protected content; the next fragment's `` would then be + // read as a real call instead of the documentation example it is. + let mut s = StreamScrubber::new(); + let first = s.feed("example:\n```bash\n"); + assert_eq!( + first.text, "example:\n", + "the open fence must be held, not drained" + ); + assert!(first.calls.is_empty()); + + let second = + s.feed("echo {\"name\":\"x\",\"arguments\":{}}\n```\nafter"); + assert!( + second.calls.is_empty(), + "the fenced example must not dispatch a call: {second:?}" + ); + assert_eq!( + second.text, + "```bash\necho {\"name\":\"x\",\"arguments\":{}}\n```\nafter" + ); +} + +#[test] +fn a_harmony_channel_header_without_message_yet_is_held() { + // The header names a target but `<|message|>` has not streamed in yet, + // so the scrubber must hold the fragment rather than guess. + let mut s = StreamScrubber::new(); + let first = s.feed("<|channel|>commentary to=functions.read"); + assert_eq!(first.text, "", "a pending channel header must be held"); + let second = s.feed("<|message|>{\"path\":\"a\"}<|call|>tail"); + assert_eq!(second.text, "tail"); + assert_eq!(second.calls[0].name, "read"); +} + +#[test] +fn stream_matches_batch_parser_on_the_visible_text() { + let full = r#"lead {"name":"a","arguments":{}} mid {"name":"b","arguments":{"k":1}} tail"#; + let (batch, calls) = parse_tool_calls(full); + assert_eq!(calls.len(), 2); + let frags: Vec = full.chars().map(|c| c.to_string()).collect(); + let refs: Vec<&str> = frags.iter().map(String::as_str).collect(); + let (streamed, stream_calls) = scrub_all(&refs); + assert_eq!( + streamed.split_whitespace().collect::>(), + batch.split_whitespace().collect::>() + ); + assert_eq!(stream_calls, 2); +} + +#[test] +fn known_tools_repair_streamed_names() { + let mut s = StreamScrubber::new().with_known_tools(vec!["read_file".into()]); + let step = s.feed("{\"name\":\"functions.read_file\",\"arguments\":{}}"); + assert_eq!(step.calls[0].name, "read_file"); +} diff --git a/crates/tinytools-agent/src/types.rs b/crates/tinytools-agent/src/types.rs new file mode 100644 index 0000000..2d2078a --- /dev/null +++ b/crates/tinytools-agent/src/types.rs @@ -0,0 +1,207 @@ +//! The vocabulary shared by every parser, repair, and renderer in this crate. + +use serde_json::Value; + +use crate::PFormatRegistry; + +/// Which grammar a call was recovered through. +/// +/// Carried on every [`ParsedToolCall`] so a host can log *how* a call reached +/// it — a run that only ever dispatches [`CallSource::Native`] calls behaves +/// very differently from one living on [`CallSource::Sentinel`] recoveries, +/// and the difference is invisible without this. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum CallSource { + /// The provider's structured tool-call channel. + Native, + /// `{json}` and its spelling variants, including + /// fenced ```` ```tool_call ```` blocks. + TaggedJson, + /// `` XML: Claude, `DeepSeek` DSML, + /// namespaced variants, and `` forms. + InvokeXml, + /// Chat-template sentinel tokens leaked verbatim: `DeepSeek`-R1 + /// `<｜tool▁call▁begin｜>` and Kimi `<|tool_call_begin|>`. + Sentinel, + /// gpt-oss Harmony `<|channel|>commentary to=…<|message|>…<|call|>`. + Harmony, + /// Mistral `[TOOL_CALLS]` blocks. + Mistral, + /// GLM `tool/param>value` lines. + Glm, + /// A whole response that is one JSON object or `tool_calls` envelope. + BareJson, + /// P-Format `name[index|value]` inside a tag. + PFormat, +} + +/// One model-requested tool invocation recovered from text or structured data. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedToolCall { + /// Tool name as the model wrote it, after name repair when a known-tool + /// set was supplied. + pub name: String, + /// Parsed tool arguments, defaulting to an empty object when absent. + pub arguments: Value, + /// Provider-assigned call id when the call came from a native + /// tool-use response. `None` for every text-recovered call — this crate + /// never mints ids, the host does, so two turns of one run can never + /// collide on a per-response counter. + pub id: Option, + /// The grammar the call was recovered through. + pub source: CallSource, +} + +impl ParsedToolCall { + /// A text-recovered call with no provider id. + #[must_use] + pub fn new(name: impl Into, arguments: Value, source: CallSource) -> Self { + Self { + name: name.into(), + arguments, + id: None, + source, + } + } + + /// A call the provider reported natively, with its id. + #[must_use] + pub fn native(id: impl Into, name: impl Into, arguments: Value) -> Self { + Self { + name: name.into(), + arguments, + id: Some(id.into()), + source: CallSource::Native, + } + } +} + +/// What the caller knows that makes parsing safer. +/// +/// Every field widens or narrows recovery. The default — no known tools, no +/// registry, bare JSON allowed — is what a caller with no context gets and is +/// safe, because the anti-phantom rules in [`crate::parse`] do not depend on +/// any of it. Supplying `known_tools` is what unlocks name repair and the +/// alias-tolerant bare-object path. +#[derive(Debug, Clone, Copy, Default)] +pub struct ParseOptions<'a> { + /// The tools the model was actually offered this turn. Enables name + /// repair (`terminal" parameter=…` → `terminal`) and lets a bare JSON + /// object naming a known tool use the argument-key aliases. + pub known_tools: &'a [String], + /// Positional layouts for the P-Format grammar. `None` disables it. + pub registry: Option<&'a PFormatRegistry>, + /// Whether a response that is *entirely* one JSON object or array may be + /// read as a call. Off for callers whose model legitimately answers in + /// JSON. + pub allow_bare_json: bool, +} + +impl<'a> ParseOptions<'a> { + /// The permissive default: bare JSON allowed, no known tools, no registry. + #[must_use] + pub fn new() -> Self { + Self { + known_tools: &[], + registry: None, + allow_bare_json: true, + } + } + + /// Sets the tools the model was offered. + #[must_use] + pub fn with_known_tools(mut self, tools: &'a [String]) -> Self { + self.known_tools = tools; + self + } + + /// Sets the P-Format registry. + #[must_use] + pub fn with_registry(mut self, registry: &'a PFormatRegistry) -> Self { + self.registry = Some(registry); + self + } + + /// Forbids the whole-response JSON path. + #[must_use] + pub fn without_bare_json(mut self) -> Self { + self.allow_bare_json = false; + self + } + + /// Whether `name` is one of the offered tools. Always `false` when no + /// tools were supplied, so callers can tell "unknown" from "unchecked" via + /// [`Self::has_known_tools`]. + #[must_use] + pub fn knows(&self, name: &str) -> bool { + self.known_tools.iter().any(|known| known == name) + } + + /// Whether a known-tool set was supplied at all. + #[must_use] + pub fn has_known_tools(&self) -> bool { + !self.known_tools.is_empty() + } +} + +/// Why a span of model output was *not* turned into a call, or was changed on +/// the way. +/// +/// Diagnostics never carry model output — only lengths and names — so a host +/// can log them at any level without leaking tool arguments. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum ParseDiagnostic { + /// A recognised block whose body did not decode into a call and was + /// dropped from the narrative. + MalformedBlock { + /// The grammar that recognised the block. + source: CallSource, + /// Length of the dropped body in characters. + body_chars: usize, + }, + /// A recognised opener with no closer; the span was kept as text. + UnterminatedBlock { + /// The grammar that recognised the opener. + source: CallSource, + }, + /// The model's tool name was rewritten to a known tool. + NameRepaired { + /// What the model wrote. + from: String, + /// What it was resolved to. + to: String, + }, + /// The tool name does not match any offered tool and could not be + /// repaired. The call is still returned; the host's unknown-tool policy + /// decides what happens to it. + UnknownTool { + /// The unresolved name. + name: String, + }, + /// Arguments were recovered from relaxed or damaged JSON. + ArgumentsRepaired { + /// The tool the arguments belong to. + tool: String, + }, +} + +/// The result of parsing one model response. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ParseOutcome { + /// The narrative text with every recognised block removed. + pub text: String, + /// The calls, in source order. + pub calls: Vec, + /// What was dropped, repaired, or left unresolved. + pub diagnostics: Vec, +} + +impl ParseOutcome { + /// Splits into the `(text, calls)` pair the dialect API speaks. + #[must_use] + pub fn into_parts(self) -> (String, Vec) { + (self.text, self.calls) + } +} diff --git a/docs/specs/agent-tool-protocols.md b/docs/specs/agent-tool-protocols.md index f1de8a7..5e19529 100644 --- a/docs/specs/agent-tool-protocols.md +++ b/docs/specs/agent-tool-protocols.md @@ -2,29 +2,37 @@ ## Purpose -TinyTools separates two reusable concerns: - -- `tinytools` describes callable tools and their results; -- `tinytools-agent` translates those declarations and results to and from the - protocols models use during an agent loop. +`tinytools-agent` is the single owner of the model-facing tool protocol for +every TinyHumans consumer: TinyAgents (host level), TinyInference (provider +level), and any host that drives its own model loop. It renders what a model +reads, parses what a model writes, repairs what a model damaged, and replays a +transcript onto the wire. See `crates/tinytools-agent/README.md` for the +module map and the grammar table. ## Ownership -`tinytools-agent` owns tolerant call parsing, P-Format registry construction, -XML/P-Format/native dialects, catalogue and result-block rendering, and -provider-neutral transcript replay repair. +| Concern | Owner | +| --- | --- | +| Tool vocabulary (`Tool`, `ToolSpec`, `ToolResult`) | `tinytools` | +| Parsing, repair, streaming scrub, rendering, dialects | `tinytools-agent` | +| Provider wire translation (native `tools` / `tool_calls`, SSE assembly, reasoning side-channels) | `tinyinference-llm` | +| Dialect selection, call-id minting, argument validation policy, execution, unknown-tool policy, re-prompt nudges | the host (`tinyagents-harness`) | -It does not own tool execution, permission or approval policy, sandboxing, -timeouts, provider transports, or an agent loop. +A format-specific string — a DSML marker, a Kimi sentinel, a Harmony channel +token — belongs in exactly one grammar file under +`crates/tinytools-agent/src/parse/grammar/`. A consumer that finds itself +matching one is looking at a bug to fix here. ## Dependency boundary -The crate consumes `tinytools::ToolSpec`. It must not depend on TinyAgents or a -provider runtime. A harness maps its message and response types onto the thin -`DialectResponse`, `TranscriptEntry`, and `DialectMessage` types. +`tinytools-agent` depends on `tinytools`, `regex`, `serde`, `serde_json` and +optionally `tracing`. It never depends on an inference runtime, a transport, +or TinyAgents. `tinyinference-llm` and `tinyagents-harness` depend on it. ## Compatibility -TinyAgents may re-export this API from its historical -`tinyagents_harness::tool_calling` module. That is a compatibility adapter, not -the implementation owner. +`parse_tool_calls`, `parse_tool_calls_with_pformat`, `parse_tool_call_value`, +`parse_tool_calls_from_json_value`, `extract_json_values`, +`parse_arguments_value`, `parse_glm_style_tool_calls` and the `dialect` API +keep their signatures. `ParsedToolCall` gained a `source: CallSource` field +and a `new` / `native` constructor; construct it through those.