From 2c3ff818f55f0644970eeaeee51b60cb9fa10100 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 02:16:24 +0300 Subject: [PATCH 1/5] feat(llm): expose provider request extensions Co-authored-by: Medulla --- crates/tinyinference-llm/src/model/types.rs | 4 + .../src/providers/anthropic/config.rs | 26 ++++-- .../src/providers/anthropic/config_test.rs | 5 ++ .../src/providers/anthropic/mod.rs | 29 ++++++- .../src/providers/anthropic/request.rs | 25 ++++-- .../src/providers/anthropic/response.rs | 6 +- .../src/providers/anthropic/stream.rs | 8 +- .../src/providers/anthropic/test.rs | 81 ++++++++++++++++++- .../src/providers/openai/test.rs | 14 +++- 9 files changed, 170 insertions(+), 28 deletions(-) diff --git a/crates/tinyinference-llm/src/model/types.rs b/crates/tinyinference-llm/src/model/types.rs index 4e51699..2301e63 100644 --- a/crates/tinyinference-llm/src/model/types.rs +++ b/crates/tinyinference-llm/src/model/types.rs @@ -84,6 +84,9 @@ pub enum ReasoningEffort { Medium, /// Above-default effort. High, + /// Maximum provider-supported effort. + #[serde(rename = "xhigh")] + XHigh, /// Explicitly disable reasoning. None, } @@ -96,6 +99,7 @@ impl ReasoningEffort { Self::Low => "low", Self::Medium => "medium", Self::High => "high", + Self::XHigh => "xhigh", Self::None => "none", } } diff --git a/crates/tinyinference-llm/src/providers/anthropic/config.rs b/crates/tinyinference-llm/src/providers/anthropic/config.rs index 93c7246..4670f4c 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/config.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/config.rs @@ -19,6 +19,8 @@ pub struct AnthropicConfig<'a> { pub temperature_override: Option, /// Model-id glob patterns whose targets reject temperature. pub temperature_unsupported_models: &'a [String], + /// Static headers attached to every request. + pub extra_headers: &'a [(String, String)], } impl std::fmt::Debug for AnthropicConfig<'_> { @@ -34,6 +36,14 @@ impl std::fmt::Debug for AnthropicConfig<'_> { "temperature_unsupported_models", &self.temperature_unsupported_models, ) + .field( + "extra_header_names", + &self + .extra_headers + .iter() + .map(|(name, _)| name) + .collect::>(), + ) .finish() } } @@ -48,12 +58,12 @@ pub fn endpoint_is_anthropic_messages(endpoint: &str) -> bool { /// Builds an Anthropic Messages model from fully resolved configuration. pub fn build_anthropic_model(config: AnthropicConfig<'_>) -> Arc> { - Arc::new( - AnthropicModel::with_base_url(config.api_key, config.endpoint) - .with_model(config.model) - .with_temperature_override(config.temperature_override) - .with_temperature_unsupported_models( - config.temperature_unsupported_models.iter().cloned(), - ), - ) + let mut model = AnthropicModel::with_base_url(config.api_key, config.endpoint) + .with_model(config.model) + .with_temperature_override(config.temperature_override) + .with_temperature_unsupported_models(config.temperature_unsupported_models.iter().cloned()); + for (name, value) in config.extra_headers { + model = model.with_header(name.clone(), value.clone()); + } + Arc::new(model) } diff --git a/crates/tinyinference-llm/src/providers/anthropic/config_test.rs b/crates/tinyinference-llm/src/providers/anthropic/config_test.rs index e380d80..5b999d9 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/config_test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/config_test.rs @@ -18,6 +18,7 @@ fn builds_a_native_anthropic_model_with_the_configured_profile() { model: "claude-sonnet-4-6", temperature_override: Some(0.2), temperature_unsupported_models: &[], + extra_headers: &[], }); let profile = model.profile().expect("anthropic models expose a profile"); assert_eq!(profile.provider.as_deref(), Some("anthropic")); @@ -31,12 +32,14 @@ fn builds_a_native_anthropic_model_with_the_configured_profile() { #[test] fn debug_redacts_api_key() { + let headers = vec![("anthropic-beta".to_string(), "secret-beta".to_string())]; let config = AnthropicConfig { endpoint: "https://endpoint-user:endpoint-pass@api.anthropic.com/v1?token=query-secret#fragment-secret", api_key: "sk-ant-secret", model: "claude-sonnet-4-6", temperature_override: None, temperature_unsupported_models: &[], + extra_headers: &headers, }; let debug = format!("{config:?}"); assert!(!debug.contains("sk-ant-secret")); @@ -44,6 +47,8 @@ fn debug_redacts_api_key() { assert!(!debug.contains("endpoint-pass")); assert!(!debug.contains("query-secret")); assert!(!debug.contains("fragment-secret")); + assert!(!debug.contains("secret-beta")); + assert!(debug.contains("anthropic-beta")); assert!(debug.contains("token")); assert!(debug.contains("[REDACTED]")); } diff --git a/crates/tinyinference-llm/src/providers/anthropic/mod.rs b/crates/tinyinference-llm/src/providers/anthropic/mod.rs index ed419f1..030b2fd 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/mod.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/mod.rs @@ -84,6 +84,7 @@ pub struct AnthropicModel { /// [`Self::with_temperature_override`]. temperature_override: Option, temperature_unsupported: Vec, + extra_headers: Vec<(String, String)>, allow_insecure_http: bool, } @@ -98,6 +99,14 @@ impl std::fmt::Debug for AnthropicModel { .field("profile", &self.profile) .field("temperature_override", &self.temperature_override) .field("temperature_unsupported", &self.temperature_unsupported) + .field( + "extra_header_names", + &self + .extra_headers + .iter() + .map(|(name, _)| name) + .collect::>(), + ) .field("allow_insecure_http", &self.allow_insecure_http) .finish() } @@ -139,6 +148,7 @@ impl AnthropicModel { model, temperature_override: None, temperature_unsupported: Vec::new(), + extra_headers: Vec::new(), allow_insecure_http: false, } } @@ -168,6 +178,16 @@ impl AnthropicModel { self } + /// Attaches a static header to every Messages API request. + /// + /// Header values are redacted from [`Debug`](std::fmt::Debug) output. + /// Headers are applied after the built-in Anthropic authentication headers, + /// allowing compatible gateways to override them when necessary. + pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { + self.extra_headers.push((name.into(), value.into())); + self + } + /// Replaces the HTTP client, so a host can supply its own transport /// (platform TLS, proxies, default headers, timeouts). pub fn with_client(mut self, client: reqwest::Client) -> Self { @@ -253,12 +273,15 @@ impl AnthropicModel { if streaming { body["stream"] = Value::Bool(true); } - let request_builder = self + let mut request_builder = self .client .post(endpoint) .header("x-api-key", &self.api_key) - .header("anthropic-version", ANTHROPIC_VERSION) - .json(&body); + .header("anthropic-version", ANTHROPIC_VERSION); + for (name, value) in &self.extra_headers { + request_builder = request_builder.header(name.as_str(), value.as_str()); + } + let request_builder = request_builder.json(&body); let request_builder = match (streaming, request.timeout_ms) { (false, Some(timeout_ms)) => request_builder.timeout(Duration::from_millis(timeout_ms)), _ => request_builder, diff --git a/crates/tinyinference-llm/src/providers/anthropic/request.rs b/crates/tinyinference-llm/src/providers/anthropic/request.rs index 53d6cfe..aa140e4 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/request.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/request.rs @@ -152,6 +152,7 @@ pub(crate) fn request_body(request: &ModelRequest, default_model: &str) -> Value ReasoningEffort::Low => "low", ReasoningEffort::Medium => "medium", ReasoningEffort::High => "high", + ReasoningEffort::XHigh => "max", ReasoningEffort::None => unreachable!(), }, }); @@ -212,9 +213,8 @@ fn text_only_blocks(content: &[ContentBlock]) -> Vec { .collect() } -/// User-side content: text and images. Thinking blocks never appear in user -/// content; provider extensions have no faithful representation and are -/// dropped. +/// User-side content: text, images, and opaque native blocks. Thinking blocks +/// never appear in user content. fn content_blocks(content: &[ContentBlock]) -> Vec { content .iter() @@ -222,9 +222,8 @@ fn content_blocks(content: &[ContentBlock]) -> Vec { ContentBlock::Text(text) => text_block(text), ContentBlock::Json(value) => text_block(&value.to_string()), ContentBlock::Image(image) => Some(image_block(image)), - ContentBlock::Thinking { .. } - | ContentBlock::RedactedThinking { .. } - | ContentBlock::ProviderExtension(_) => None, + ContentBlock::ProviderExtension(value) => provider_extension_block(value), + ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => None, }) .collect() } @@ -250,15 +249,25 @@ fn assistant_blocks(content: &[ContentBlock]) -> Vec { ContentBlock::RedactedThinking { data } => { Some(json!({ "type": "redacted_thinking", "data": data })) } + ContentBlock::ProviderExtension(value) => provider_extension_block(value), ContentBlock::Thinking { signature: None, .. } - | ContentBlock::Image(_) - | ContentBlock::ProviderExtension(_) => None, + | ContentBlock::Image(_) => None, }) .collect() } +/// Returns an opaque Anthropic content block when it has the object shape the +/// Messages API requires. Keeping the full object intact lets hosts persist and +/// replay newer block types without waiting for a TinyInference release. +fn provider_extension_block(value: &Value) -> Option { + value + .as_object() + .filter(|object| object.get("type").is_some_and(Value::is_string)) + .map(|_| value.clone()) +} + /// Renders an image reference: a `data:` URI becomes an inline base64 source, /// anything else a URL source. fn image_block(image: &ImageRef) -> Value { diff --git a/crates/tinyinference-llm/src/providers/anthropic/response.rs b/crates/tinyinference-llm/src/providers/anthropic/response.rs index 67c1497..069d9f9 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/response.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/response.rs @@ -70,11 +70,7 @@ pub(crate) fn parse_response(body: Value) -> Result { "redacted_thinking" => content.push(ContentBlock::RedactedThinking { data: required_string(block.get("data"), "content[].data")?.to_string(), }), - other => { - return Err(malformed(&format!( - "unsupported content block type: {other}" - ))); - } + _ => content.push(ContentBlock::ProviderExtension(block.clone())), } } Ok(ModelResponse { diff --git a/crates/tinyinference-llm/src/providers/anthropic/stream.rs b/crates/tinyinference-llm/src/providers/anthropic/stream.rs index 175056d..69dc8a6 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/stream.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/stream.rs @@ -53,6 +53,7 @@ enum OpenBlock { signature: Option, }, Redacted(String), + ProviderExtension(Value), } /// Provider-side accumulator rebuilding the terminal [`ModelResponse`]. @@ -137,7 +138,7 @@ impl AnthropicStreamAcc { Some("redacted_thinking") => { OpenBlock::Redacted(block["data"].as_str().unwrap_or_default().to_string()) } - _ => { + Some("text") => { let text = block["text"].as_str().unwrap_or_default().to_string(); if !text.is_empty() { pending.push_back(ModelStreamItem::MessageDelta(MessageDelta::text( @@ -146,6 +147,7 @@ impl AnthropicStreamAcc { } OpenBlock::Text(text) } + _ => OpenBlock::ProviderExtension(block.clone()), }; *self.slot(index) = Some(open); } @@ -245,6 +247,9 @@ impl AnthropicStreamAcc { content.push(ContentBlock::Thinking { text, signature }); } OpenBlock::Redacted(data) => content.push(ContentBlock::RedactedThinking { data }), + OpenBlock::ProviderExtension(value) => { + content.push(ContentBlock::ProviderExtension(value)); + } OpenBlock::ToolUse { id, name, @@ -277,6 +282,7 @@ impl AnthropicStreamAcc { "stop_reason": self.stop_reason, "content": content.iter().filter_map(|block| match block { ContentBlock::Text(text) => Some(serde_json::json!({"type": "text", "text": text})), + ContentBlock::ProviderExtension(value) => Some(value.clone()), _ => None, }).chain(tool_calls.iter().map(|call| serde_json::json!({ "type": "tool_use", "id": call.id, "name": call.name, "input": call.arguments, diff --git a/crates/tinyinference-llm/src/providers/anthropic/test.rs b/crates/tinyinference-llm/src/providers/anthropic/test.rs index 209bca1..f8963b4 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/test.rs @@ -347,6 +347,52 @@ fn normalized_reasoning_is_lowered_to_anthropic_thinking() { let body = request_body(&adaptive, "m"); assert_eq!(body["thinking"], json!({ "type": "adaptive" })); assert_eq!(body["output_config"]["effort"], "high"); + + let maximum = + ModelRequest::new(vec![Message::user("hi")]).with_reasoning_effort(ReasoningEffort::XHigh); + let body = request_body(&maximum, "m"); + assert_eq!(body["output_config"]["effort"], "max"); +} + +#[test] +fn provider_extension_blocks_round_trip_without_interpretation() { + let extension = json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": { "query": "rust" }, + "future_field": true, + }); + let response = parse_response(json!({ + "id": "msg_ext", + "content": [extension.clone()], + "stop_reason": "end_turn", + "usage": { "input_tokens": 1, "output_tokens": 1 } + })) + .unwrap(); + assert_eq!( + response.message.content, + vec![ContentBlock::ProviderExtension(extension.clone())] + ); + + let replay = request_body( + &ModelRequest::new(vec![Message::Assistant(response.message)]), + "m", + ); + assert_eq!(replay["messages"][0]["content"][0], extension); +} + +#[test] +fn malformed_provider_extension_blocks_are_not_sent() { + let request = ModelRequest::new(vec![Message::User(crate::message::UserMessage { + content: vec![ContentBlock::ProviderExtension(json!({"opaque": true}))], + })]); + assert!( + request_body(&request, "m")["messages"] + .as_array() + .unwrap() + .is_empty() + ); } #[test] @@ -497,10 +543,17 @@ fn temperature_policy_uses_the_effective_request_model() { #[test] fn debug_redacts_the_api_key() { - let model = AnthropicModel::new("secret-api-key"); + let model = + AnthropicModel::new("secret-api-key").with_header("anthropic-beta", "secret-beta-value"); let debug = format!("{model:?}"); assert!(debug.contains("[redacted]")); assert!(!debug.contains("secret-api-key")); + assert!(!debug.contains("secret-beta-value")); + assert!(debug.contains("anthropic-beta")); + assert_eq!( + model.extra_headers, + vec![("anthropic-beta".into(), "secret-beta-value".into())] + ); } fn sse(events: &[serde_json::Value]) -> Vec { @@ -606,6 +659,32 @@ async fn streaming_thinking_arrives_on_the_reasoning_channel_with_its_signature( )); } +#[tokio::test] +async fn streaming_preserves_unknown_content_blocks_for_the_host() { + let extension = json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": { "query": "rust" } + }); + let events = [ + json!({"type":"message_start","message":{"id":"m","usage":{"input_tokens":1,"output_tokens":0}}}), + json!({"type":"content_block_start","index":0,"content_block":extension.clone()}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}), + json!({"type":"message_stop"}), + ]; + let items: Vec = stream::stream_from_bytes(vec![sse(&events)], "m") + .collect() + .await; + let Some(ModelStreamItem::Completed(response)) = items.last() else { + panic!("stream must end in Completed, got {:?}", items.last()); + }; + assert_eq!( + response.message.content, + vec![ContentBlock::ProviderExtension(extension)] + ); +} + #[tokio::test] async fn streaming_without_message_stop_is_a_provider_failure() { let events = [ diff --git a/crates/tinyinference-llm/src/providers/openai/test.rs b/crates/tinyinference-llm/src/providers/openai/test.rs index 400e2eb..21b4ff4 100644 --- a/crates/tinyinference-llm/src/providers/openai/test.rs +++ b/crates/tinyinference-llm/src/providers/openai/test.rs @@ -10,8 +10,8 @@ use serde_json::json; use super::*; use crate::message::Message; use crate::model::{ - ChatModel, ModelRequest, ModelStreamItem, ProviderError, ResponseFormat, StreamAccumulator, - ToolChoice, + ChatModel, ModelRequest, ModelStreamItem, ProviderError, ReasoningEffort, ResponseFormat, + StreamAccumulator, ToolChoice, }; use crate::providers::{ProviderKind, ProviderSpec}; use crate::tool::ToolSchema; @@ -109,6 +109,16 @@ fn translates_request_to_openai_json_shape() { assert_eq!(value["seed"], json!(7)); } +#[test] +fn translates_maximum_reasoning_effort() { + let request = ModelRequest::new(vec![Message::user("solve")]) + .with_model("o3") + .with_reasoning_effort(ReasoningEffort::XHigh); + let body = model().translate_request(&request).unwrap(); + let value = serde_json::to_value(body).unwrap(); + assert_eq!(value["reasoning_effort"], "xhigh"); +} + #[test] fn translates_provider_options_for_local_openai_compatible_models() { let request = ModelRequest::new(vec![Message::user("hi")]) From 6392a924d9f78eae9443102291eff8d6dcd86b3f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 10:41:40 +0300 Subject: [PATCH 2/5] feat(anthropic): handle provider extension blocks in streaming Replace the strict validation of provider extension blocks with a permissive approach that accepts any block type, and introduce a `ProviderExtension` variant in `BlockKind` to expose the block type to consumers. In the streaming path, accumulate `input_json_delta` fragments as raw JSON strings and parse them only when the block is closed, fixing a bug where partial JSON updates were incorrectly merged into the block's input object. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyinference-llm/src/model/types.rs | 5 ++ .../src/providers/anthropic/request.rs | 22 ++------ .../src/providers/anthropic/stream.rs | 51 ++++++++++++------- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/crates/tinyinference-llm/src/model/types.rs b/crates/tinyinference-llm/src/model/types.rs index 61ec39a..7dbcf3e 100644 --- a/crates/tinyinference-llm/src/model/types.rs +++ b/crates/tinyinference-llm/src/model/types.rs @@ -921,6 +921,11 @@ pub enum BlockKind { /// Tool name. name: String, }, + /// An opaque provider-defined content block. + ProviderExtension { + /// Provider wire type for the extension block. + block_type: String, + }, } /// An incremental fragment belonging to the open block named in the diff --git a/crates/tinyinference-llm/src/providers/anthropic/request.rs b/crates/tinyinference-llm/src/providers/anthropic/request.rs index dbd5751..cf60d7d 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/request.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/request.rs @@ -275,24 +275,10 @@ fn assistant_blocks(content: &[ContentBlock]) -> Vec { /// replay newer block types without waiting for a TinyInference release. fn provider_extension_block(value: &Value) -> Option { let object = value.as_object()?; - let kind = object.get("type")?.as_str()?; - let valid = match kind { - "text" => object.get("text").is_some_and(Value::is_string), - "server_tool_use" => { - object.get("id").is_some_and(Value::is_string) - && object.get("name").is_some_and(Value::is_string) - && object.get("input").is_some_and(Value::is_object) - } - "tool_use" => { - object.get("id").is_some_and(Value::is_string) - && object.get("name").is_some_and(Value::is_string) - && object.get("input").is_some_and(Value::is_object) - } - "tool_result" => object.get("tool_use_id").is_some_and(Value::is_string), - "image" | "document" | "redacted_thinking" | "thinking" => true, - _ => false, - }; - valid.then(|| value.clone()) + object + .get("type") + .and_then(Value::as_str) + .map(|_| value.clone()) } /// Renders an image reference: a `data:` URI becomes an inline base64 source, diff --git a/crates/tinyinference-llm/src/providers/anthropic/stream.rs b/crates/tinyinference-llm/src/providers/anthropic/stream.rs index f1bd0c5..032d449 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/stream.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/stream.rs @@ -55,7 +55,10 @@ enum OpenBlock { signature: Option, }, Redacted(String), - ProviderExtension(Value), + ProviderExtension { + value: Value, + partial_json: String, + }, } impl OpenBlock { @@ -88,7 +91,17 @@ impl OpenBlock { "arguments": arguments, })) } - OpenBlock::ProviderExtension(value) => ContentBlock::ProviderExtension(value), + OpenBlock::ProviderExtension { + mut value, + partial_json, + } => { + if !partial_json.trim().is_empty() + && let Ok(input) = serde_json::from_str::(&partial_json) + { + value["input"] = input; + } + ContentBlock::ProviderExtension(value) + } } } } @@ -220,7 +233,18 @@ impl AnthropicStreamAcc { } OpenBlock::Text(text) } - _ => OpenBlock::ProviderExtension(block.clone()), + _ => { + pending.push_back(ModelStreamItem::BlockStart { + index, + kind: BlockKind::ProviderExtension { + block_type: block["type"].as_str().unwrap_or_default().to_string(), + }, + }); + OpenBlock::ProviderExtension { + value: block.clone(), + partial_json: String::new(), + } + } }; *self.slot(index) = Some(open); } @@ -261,21 +285,12 @@ impl AnthropicStreamAcc { content_index: Some(index), })); } - (Some("input_json_delta"), Some(OpenBlock::ProviderExtension(value))) => { + ( + Some("input_json_delta"), + Some(OpenBlock::ProviderExtension { partial_json, .. }), + ) => { let fragment = delta["partial_json"].as_str().unwrap_or_default(); - if let Ok(parsed) = serde_json::from_str::(fragment) { - if let Some(object) = parsed.as_object() { - if let Some(input_object) = value["input"].as_object_mut() { - for (key, item) in object { - input_object.insert(key.clone(), item.clone()); - } - } else { - value["input"] = parsed; - } - } else { - value["input"] = parsed; - } - } + partial_json.push_str(fragment); } (Some("thinking_delta"), Some(OpenBlock::Thinking { text, .. })) => { let fragment = delta["thinking"].as_str().unwrap_or_default(); @@ -358,7 +373,7 @@ impl AnthropicStreamAcc { content.push(ContentBlock::Thinking { text, signature }); } OpenBlock::Redacted(data) => content.push(ContentBlock::RedactedThinking { data }), - OpenBlock::ProviderExtension(value) => { + OpenBlock::ProviderExtension { value, .. } => { content.push(ContentBlock::ProviderExtension(value)); } OpenBlock::ToolUse { From 7dd207af44188ec57c500084558ddf84cf7c6cd7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 10:42:18 +0300 Subject: [PATCH 3/5] test(anthropic): add streaming test for extension input reconstruction Add a test that verifies the streaming path correctly reassembles fragmented JSON input for provider extension blocks and preserves the boundaries between content block start, delta, and stop events. This ensures that server-tool-use blocks with partial JSON deltas are reconstructed into complete provider extension blocks with the correct input. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/providers/anthropic/test.rs | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/anthropic/test.rs b/crates/tinyinference-llm/src/providers/anthropic/test.rs index 7f0b082..7e6a59d 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/test.rs @@ -453,10 +453,20 @@ fn provider_extension_blocks_round_trip_without_interpretation() { ); let replay = request_body( - &ModelRequest::new(vec![Message::Assistant(response.message)]), + &ModelRequest::new(vec![Message::Assistant(response.message.clone())]), "m", ); assert_eq!(replay["messages"][0]["content"][0], extension); + + let future_extension = json!({ "type": "future_block", "payload": { "ok": true } }); + let replay = request_body( + &ModelRequest::new(vec![Message::Assistant(AssistantMessage { + content: vec![ContentBlock::ProviderExtension(future_extension.clone())], + ..response.message + })]), + "m", + ); + assert_eq!(replay["messages"][0]["content"][0], future_extension); } #[test] @@ -762,6 +772,37 @@ async fn streaming_preserves_unknown_content_blocks_for_the_host() { ); } +#[tokio::test] +async fn streaming_reconstructs_extension_input_fragments_and_boundaries() { + let events = [ + json!({"type":"message_start","message":{"id":"m","usage":{"input_tokens":1,"output_tokens":0}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{}}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":"}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"rust\"}"}}), + json!({"type":"content_block_stop","index":0}), + json!({"type":"message_stop"}), + ]; + let items: Vec = stream::stream_from_bytes(vec![sse(&events)], "m") + .collect() + .await; + + assert!(items.iter().any(|item| matches!( + item, + ModelStreamItem::BlockStart { + index: 0, + kind: BlockKind::ProviderExtension { block_type } + } if block_type == "server_tool_use" + ))); + assert!(items.iter().any(|item| matches!( + item, + ModelStreamItem::BlockEnd { + index: 0, + block: ContentBlock::ProviderExtension(value) + } if value["input"] == json!({"query": "rust"}) + ))); + assert!(matches!(items.last(), Some(ModelStreamItem::Completed(_)))); +} + #[tokio::test] async fn streaming_without_message_stop_is_a_provider_failure() { let events = [ From 46f292ee76e69651803bce678676f0074bbbfede Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 10:42:28 +0300 Subject: [PATCH 4/5] fix(anthropic): add missing AssistantMessage import in test The test module for the Anthropic provider was missing an import for `AssistantMessage`, which is now used in the test code. This change adds the missing import to resolve the compilation error. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyinference-llm/src/providers/anthropic/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyinference-llm/src/providers/anthropic/test.rs b/crates/tinyinference-llm/src/providers/anthropic/test.rs index 7e6a59d..158d06f 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/test.rs @@ -3,7 +3,7 @@ use serde_json::json; use super::*; use crate::cache::CachePolicy; -use crate::message::{ContentBlock, ImageRef, Message, ToolMessage}; +use crate::message::{AssistantMessage, ContentBlock, ImageRef, Message, ToolMessage}; use crate::model::{ BlockDelta, BlockKind, ModelStreamItem, PromptSegment, ReasoningConfig, ReasoningEffort, SegmentRole, ToolChoice, From 55875a6fd65fbb28d461019c8b342717d7f7a9a0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 20 Sep 2026 10:46:51 +0300 Subject: [PATCH 5/5] fix(anthropic): reject native block types in provider extension handling The provider extension function now filters out known native block types such as text, image, document, tool_use, thinking, and redacted_thinking, preventing callers from bypassing normalized representations with incomplete provider-shaped JSON. The streaming path also consolidates the partial JSON merging logic into a shared helper function to ensure consistent behavior when reconstructing extension blocks. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/providers/anthropic/request.rs | 13 +++++--- .../src/providers/anthropic/stream.rs | 30 ++++++++++++------- .../src/providers/anthropic/test.rs | 21 +++++++++++++ 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/anthropic/request.rs b/crates/tinyinference-llm/src/providers/anthropic/request.rs index cf60d7d..9dc3c46 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/request.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/request.rs @@ -275,10 +275,15 @@ fn assistant_blocks(content: &[ContentBlock]) -> Vec { /// replay newer block types without waiting for a TinyInference release. fn provider_extension_block(value: &Value) -> Option { let object = value.as_object()?; - object - .get("type") - .and_then(Value::as_str) - .map(|_| value.clone()) + let block_type = object.get("type").and_then(Value::as_str)?; + // Provider extensions are for block types this adapter does not model. + // Rejecting native types prevents callers from bypassing their normalized + // representations with incomplete provider-shaped JSON. + (!matches!( + block_type, + "text" | "image" | "document" | "tool_use" | "thinking" | "redacted_thinking" + )) + .then(|| value.clone()) } /// Renders an image reference: a `data:` URI becomes an inline base64 source, diff --git a/crates/tinyinference-llm/src/providers/anthropic/stream.rs b/crates/tinyinference-llm/src/providers/anthropic/stream.rs index 032d449..4c0037c 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/stream.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/stream.rs @@ -92,20 +92,22 @@ impl OpenBlock { })) } OpenBlock::ProviderExtension { - mut value, + value, partial_json, - } => { - if !partial_json.trim().is_empty() - && let Ok(input) = serde_json::from_str::(&partial_json) - { - value["input"] = input; - } - ContentBlock::ProviderExtension(value) - } + } => ContentBlock::ProviderExtension(provider_extension_value(value, partial_json)), } } } +fn provider_extension_value(mut value: Value, partial_json: String) -> Value { + if !partial_json.trim().is_empty() + && let Ok(input) = serde_json::from_str::(&partial_json) + { + value["input"] = input; + } + value +} + /// Provider-side accumulator rebuilding the terminal [`ModelResponse`]. #[derive(Debug, Default)] struct AnthropicStreamAcc { @@ -373,8 +375,14 @@ impl AnthropicStreamAcc { content.push(ContentBlock::Thinking { text, signature }); } OpenBlock::Redacted(data) => content.push(ContentBlock::RedactedThinking { data }), - OpenBlock::ProviderExtension { value, .. } => { - content.push(ContentBlock::ProviderExtension(value)); + OpenBlock::ProviderExtension { + value, + partial_json, + } => { + content.push(ContentBlock::ProviderExtension(provider_extension_value( + value, + partial_json, + ))); } OpenBlock::ToolUse { id, diff --git a/crates/tinyinference-llm/src/providers/anthropic/test.rs b/crates/tinyinference-llm/src/providers/anthropic/test.rs index 158d06f..d575c7e 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/test.rs @@ -480,6 +480,16 @@ fn malformed_provider_extension_blocks_are_not_sent() { .unwrap() .is_empty() ); + + let native_block = ModelRequest::new(vec![Message::User(crate::message::UserMessage { + content: vec![ContentBlock::ProviderExtension(json!({"type": "text"}))], + })]); + assert!( + request_body(&native_block, "m")["messages"] + .as_array() + .unwrap() + .is_empty() + ); } #[test] @@ -800,6 +810,17 @@ async fn streaming_reconstructs_extension_input_fragments_and_boundaries() { block: ContentBlock::ProviderExtension(value) } if value["input"] == json!({"query": "rust"}) ))); + assert!(matches!( + items.last(), + Some(ModelStreamItem::Completed(response)) + if response.message.content + == vec![ContentBlock::ProviderExtension(json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "rust"} + }))] + )); assert!(matches!(items.last(), Some(ModelStreamItem::Completed(_)))); }