diff --git a/crates/tinyinference-llm/src/model/types.rs b/crates/tinyinference-llm/src/model/types.rs index e3b27a9..7dbcf3e 100644 --- a/crates/tinyinference-llm/src/model/types.rs +++ b/crates/tinyinference-llm/src/model/types.rs @@ -84,6 +84,9 @@ pub enum ReasoningEffort { Medium, /// Above-default effort. High, + /// Maximum provider-supported effort. + #[serde(rename = "xhigh")] + XHigh, /// Explicitly disable reasoning. None, } @@ -96,6 +99,7 @@ impl ReasoningEffort { Self::Low => "low", Self::Medium => "medium", Self::High => "high", + Self::XHigh => "xhigh", Self::None => "none", } } @@ -917,6 +921,11 @@ pub enum BlockKind { /// Tool name. name: String, }, + /// An opaque provider-defined content block. + ProviderExtension { + /// Provider wire type for the extension block. + block_type: String, + }, } /// An incremental fragment belonging to the open block named in the diff --git a/crates/tinyinference-llm/src/providers/anthropic/config.rs b/crates/tinyinference-llm/src/providers/anthropic/config.rs index 93c7246..4670f4c 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/config.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/config.rs @@ -19,6 +19,8 @@ pub struct AnthropicConfig<'a> { pub temperature_override: Option, /// Model-id glob patterns whose targets reject temperature. pub temperature_unsupported_models: &'a [String], + /// Static headers attached to every request. + pub extra_headers: &'a [(String, String)], } impl std::fmt::Debug for AnthropicConfig<'_> { @@ -34,6 +36,14 @@ impl std::fmt::Debug for AnthropicConfig<'_> { "temperature_unsupported_models", &self.temperature_unsupported_models, ) + .field( + "extra_header_names", + &self + .extra_headers + .iter() + .map(|(name, _)| name) + .collect::>(), + ) .finish() } } @@ -48,12 +58,12 @@ pub fn endpoint_is_anthropic_messages(endpoint: &str) -> bool { /// Builds an Anthropic Messages model from fully resolved configuration. pub fn build_anthropic_model(config: AnthropicConfig<'_>) -> Arc> { - Arc::new( - AnthropicModel::with_base_url(config.api_key, config.endpoint) - .with_model(config.model) - .with_temperature_override(config.temperature_override) - .with_temperature_unsupported_models( - config.temperature_unsupported_models.iter().cloned(), - ), - ) + let mut model = AnthropicModel::with_base_url(config.api_key, config.endpoint) + .with_model(config.model) + .with_temperature_override(config.temperature_override) + .with_temperature_unsupported_models(config.temperature_unsupported_models.iter().cloned()); + for (name, value) in config.extra_headers { + model = model.with_header(name.clone(), value.clone()); + } + Arc::new(model) } diff --git a/crates/tinyinference-llm/src/providers/anthropic/config_test.rs b/crates/tinyinference-llm/src/providers/anthropic/config_test.rs index e380d80..5b999d9 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/config_test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/config_test.rs @@ -18,6 +18,7 @@ fn builds_a_native_anthropic_model_with_the_configured_profile() { model: "claude-sonnet-4-6", temperature_override: Some(0.2), temperature_unsupported_models: &[], + extra_headers: &[], }); let profile = model.profile().expect("anthropic models expose a profile"); assert_eq!(profile.provider.as_deref(), Some("anthropic")); @@ -31,12 +32,14 @@ fn builds_a_native_anthropic_model_with_the_configured_profile() { #[test] fn debug_redacts_api_key() { + let headers = vec![("anthropic-beta".to_string(), "secret-beta".to_string())]; let config = AnthropicConfig { endpoint: "https://endpoint-user:endpoint-pass@api.anthropic.com/v1?token=query-secret#fragment-secret", api_key: "sk-ant-secret", model: "claude-sonnet-4-6", temperature_override: None, temperature_unsupported_models: &[], + extra_headers: &headers, }; let debug = format!("{config:?}"); assert!(!debug.contains("sk-ant-secret")); @@ -44,6 +47,8 @@ fn debug_redacts_api_key() { assert!(!debug.contains("endpoint-pass")); assert!(!debug.contains("query-secret")); assert!(!debug.contains("fragment-secret")); + assert!(!debug.contains("secret-beta")); + assert!(debug.contains("anthropic-beta")); assert!(debug.contains("token")); assert!(debug.contains("[REDACTED]")); } diff --git a/crates/tinyinference-llm/src/providers/anthropic/mod.rs b/crates/tinyinference-llm/src/providers/anthropic/mod.rs index e9ff512..a363498 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/mod.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/mod.rs @@ -93,6 +93,7 @@ pub struct AnthropicModel { /// [`Self::with_temperature_override`]. temperature_override: Option, temperature_unsupported: Vec, + extra_headers: Vec<(String, String)>, allow_insecure_http: bool, request_options: crate::providers::ProviderRequestOptions, } @@ -108,6 +109,14 @@ impl std::fmt::Debug for AnthropicModel { .field("profile", &self.profile) .field("temperature_override", &self.temperature_override) .field("temperature_unsupported", &self.temperature_unsupported) + .field( + "extra_header_names", + &self + .extra_headers + .iter() + .map(|(name, _)| name) + .collect::>(), + ) .field("allow_insecure_http", &self.allow_insecure_http) .field("request_options", &self.request_options) .finish() @@ -154,6 +163,7 @@ impl AnthropicModel { model, temperature_override: None, temperature_unsupported: Vec::new(), + extra_headers: Vec::new(), allow_insecure_http: false, request_options: crate::providers::ProviderRequestOptions::default(), } @@ -196,6 +206,16 @@ impl AnthropicModel { self } + /// Attaches a static header to every Messages API request. + /// + /// Header values are redacted from [`Debug`](std::fmt::Debug) output. + /// Headers are applied after the built-in Anthropic authentication headers, + /// allowing compatible gateways to override them when necessary. + pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { + self.extra_headers.push((name.into(), value.into())); + self + } + /// Replaces the HTTP client, so a host can supply its own transport /// (platform TLS, proxies, default headers, timeouts). pub fn with_client(mut self, client: reqwest::Client) -> Self { @@ -296,11 +316,24 @@ impl AnthropicModel { } self.request_options.apply_payload(&mut body); let client = self.request_options.http.as_ref().unwrap_or(&self.client); - let request_builder = client - .post(endpoint) - .header("x-api-key", &self.api_key) - .header("anthropic-version", ANTHROPIC_VERSION) - .json(&body); + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + "x-api-key", + reqwest::header::HeaderValue::from_str(&self.api_key) + .map_err(|e| Error::Validation(e.to_string()))?, + ); + headers.insert( + "anthropic-version", + reqwest::header::HeaderValue::from_static(ANTHROPIC_VERSION), + ); + for (name, value) in &self.extra_headers { + let name = reqwest::header::HeaderName::from_bytes(name.as_bytes()) + .map_err(|e| Error::Validation(e.to_string()))?; + let value = reqwest::header::HeaderValue::from_str(value) + .map_err(|e| Error::Validation(e.to_string()))?; + headers.insert(name, value); + } + let request_builder = client.post(endpoint).headers(headers).json(&body); let request_builder = match (streaming, request.timeout_ms) { (false, Some(timeout_ms)) => request_builder.timeout(Duration::from_millis(timeout_ms)), _ => request_builder, diff --git a/crates/tinyinference-llm/src/providers/anthropic/request.rs b/crates/tinyinference-llm/src/providers/anthropic/request.rs index 7c76edb..9dc3c46 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/request.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/request.rs @@ -154,6 +154,7 @@ pub(crate) fn request_body(request: &ModelRequest, default_model: &str) -> Value ReasoningEffort::Low => "low", ReasoningEffort::Medium => "medium", ReasoningEffort::High => "high", + ReasoningEffort::XHigh => "max", ReasoningEffort::None => unreachable!(), }, }); @@ -229,12 +230,11 @@ fn content_blocks(content: &[ContentBlock]) -> Vec { ContentBlock::Text(text) => text_block(text), ContentBlock::Json(value) => text_block(&value.to_string()), ContentBlock::Image(image) => Some(image_block(image)), + ContentBlock::ProviderExtension(value) => provider_extension_block(value), ContentBlock::Document(media) => Some(document_block(media)), ContentBlock::Audio(media) => Some(unsupported_media_placeholder("audio", media)), ContentBlock::Video(media) => Some(unsupported_media_placeholder("video", media)), - ContentBlock::Thinking { .. } - | ContentBlock::RedactedThinking { .. } - | ContentBlock::ProviderExtension(_) => None, + ContentBlock::Thinking { .. } | ContentBlock::RedactedThinking { .. } => None, }) .collect() } @@ -260,18 +260,32 @@ fn assistant_blocks(content: &[ContentBlock]) -> Vec { ContentBlock::RedactedThinking { data } => { Some(json!({ "type": "redacted_thinking", "data": data })) } + ContentBlock::ProviderExtension(value) => provider_extension_block(value), ContentBlock::Thinking { signature: None, .. } - | ContentBlock::Image(_) - | ContentBlock::ProviderExtension(_) - | ContentBlock::Audio(_) - | ContentBlock::Video(_) - | ContentBlock::Document(_) => None, + | ContentBlock::Image(_) => None, + ContentBlock::Audio(_) | ContentBlock::Video(_) | ContentBlock::Document(_) => None, }) .collect() } +/// Returns an opaque Anthropic content block when it has the object shape the +/// Messages API requires. Keeping the full object intact lets hosts persist and +/// replay newer block types without waiting for a TinyInference release. +fn provider_extension_block(value: &Value) -> Option { + let object = value.as_object()?; + let block_type = object.get("type").and_then(Value::as_str)?; + // Provider extensions are for block types this adapter does not model. + // Rejecting native types prevents callers from bypassing their normalized + // representations with incomplete provider-shaped JSON. + (!matches!( + block_type, + "text" | "image" | "document" | "tool_use" | "thinking" | "redacted_thinking" + )) + .then(|| value.clone()) +} + /// Renders an image reference: a `data:` URI becomes an inline base64 source, /// anything else a URL source. fn image_block(image: &ImageRef) -> Value { diff --git a/crates/tinyinference-llm/src/providers/anthropic/response.rs b/crates/tinyinference-llm/src/providers/anthropic/response.rs index 94b2cb7..27e4ed0 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/response.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/response.rs @@ -70,11 +70,7 @@ pub(crate) fn parse_response(body: Value) -> Result { "redacted_thinking" => content.push(ContentBlock::RedactedThinking { data: required_string(block.get("data"), "content[].data")?.to_string(), }), - other => { - return Err(malformed(&format!( - "unsupported content block type: {other}" - ))); - } + _ => content.push(ContentBlock::ProviderExtension(block.clone())), } } Ok(ModelResponse { diff --git a/crates/tinyinference-llm/src/providers/anthropic/stream.rs b/crates/tinyinference-llm/src/providers/anthropic/stream.rs index aacd59a..4c0037c 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/stream.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/stream.rs @@ -55,6 +55,10 @@ enum OpenBlock { signature: Option, }, Redacted(String), + ProviderExtension { + value: Value, + partial_json: String, + }, } impl OpenBlock { @@ -87,10 +91,23 @@ impl OpenBlock { "arguments": arguments, })) } + OpenBlock::ProviderExtension { + value, + partial_json, + } => ContentBlock::ProviderExtension(provider_extension_value(value, partial_json)), } } } +fn provider_extension_value(mut value: Value, partial_json: String) -> Value { + if !partial_json.trim().is_empty() + && let Ok(input) = serde_json::from_str::(&partial_json) + { + value["input"] = input; + } + value +} + /// Provider-side accumulator rebuilding the terminal [`ModelResponse`]. #[derive(Debug, Default)] struct AnthropicStreamAcc { @@ -201,7 +218,7 @@ impl AnthropicStreamAcc { }); OpenBlock::Redacted(block["data"].as_str().unwrap_or_default().to_string()) } - _ => { + Some("text") => { pending.push_back(ModelStreamItem::BlockStart { index, kind: BlockKind::Text, @@ -218,6 +235,18 @@ impl AnthropicStreamAcc { } OpenBlock::Text(text) } + _ => { + pending.push_back(ModelStreamItem::BlockStart { + index, + kind: BlockKind::ProviderExtension { + block_type: block["type"].as_str().unwrap_or_default().to_string(), + }, + }); + OpenBlock::ProviderExtension { + value: block.clone(), + partial_json: String::new(), + } + } }; *self.slot(index) = Some(open); } @@ -258,6 +287,13 @@ impl AnthropicStreamAcc { content_index: Some(index), })); } + ( + Some("input_json_delta"), + Some(OpenBlock::ProviderExtension { partial_json, .. }), + ) => { + let fragment = delta["partial_json"].as_str().unwrap_or_default(); + partial_json.push_str(fragment); + } (Some("thinking_delta"), Some(OpenBlock::Thinking { text, .. })) => { let fragment = delta["thinking"].as_str().unwrap_or_default(); text.push_str(fragment); @@ -339,6 +375,15 @@ impl AnthropicStreamAcc { content.push(ContentBlock::Thinking { text, signature }); } OpenBlock::Redacted(data) => content.push(ContentBlock::RedactedThinking { data }), + OpenBlock::ProviderExtension { + value, + partial_json, + } => { + content.push(ContentBlock::ProviderExtension(provider_extension_value( + value, + partial_json, + ))); + } OpenBlock::ToolUse { id, name, @@ -371,6 +416,7 @@ impl AnthropicStreamAcc { "stop_reason": self.stop_reason, "content": content.iter().filter_map(|block| match block { ContentBlock::Text(text) => Some(serde_json::json!({"type": "text", "text": text})), + ContentBlock::ProviderExtension(value) => Some(value.clone()), _ => None, }).chain(tool_calls.iter().map(|call| serde_json::json!({ "type": "tool_use", "id": call.id, "name": call.name, "input": call.arguments, diff --git a/crates/tinyinference-llm/src/providers/anthropic/test.rs b/crates/tinyinference-llm/src/providers/anthropic/test.rs index a91a8e1..d575c7e 100644 --- a/crates/tinyinference-llm/src/providers/anthropic/test.rs +++ b/crates/tinyinference-llm/src/providers/anthropic/test.rs @@ -3,7 +3,7 @@ use serde_json::json; use super::*; use crate::cache::CachePolicy; -use crate::message::{ContentBlock, ImageRef, Message, ToolMessage}; +use crate::message::{AssistantMessage, ContentBlock, ImageRef, Message, ToolMessage}; use crate::model::{ BlockDelta, BlockKind, ModelStreamItem, PromptSegment, ReasoningConfig, ReasoningEffort, SegmentRole, ToolChoice, @@ -424,6 +424,72 @@ fn normalized_reasoning_is_lowered_to_anthropic_thinking() { let body = request_body(&adaptive, "m"); assert_eq!(body["thinking"], json!({ "type": "adaptive" })); assert_eq!(body["output_config"]["effort"], "high"); + + let maximum = + ModelRequest::new(vec![Message::user("hi")]).with_reasoning_effort(ReasoningEffort::XHigh); + let body = request_body(&maximum, "m"); + assert_eq!(body["output_config"]["effort"], "max"); +} + +#[test] +fn provider_extension_blocks_round_trip_without_interpretation() { + let extension = json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": { "query": "rust" }, + "future_field": true, + }); + let response = parse_response(json!({ + "id": "msg_ext", + "content": [extension.clone()], + "stop_reason": "end_turn", + "usage": { "input_tokens": 1, "output_tokens": 1 } + })) + .unwrap(); + assert_eq!( + response.message.content, + vec![ContentBlock::ProviderExtension(extension.clone())] + ); + + let replay = request_body( + &ModelRequest::new(vec![Message::Assistant(response.message.clone())]), + "m", + ); + assert_eq!(replay["messages"][0]["content"][0], extension); + + let future_extension = json!({ "type": "future_block", "payload": { "ok": true } }); + let replay = request_body( + &ModelRequest::new(vec![Message::Assistant(AssistantMessage { + content: vec![ContentBlock::ProviderExtension(future_extension.clone())], + ..response.message + })]), + "m", + ); + assert_eq!(replay["messages"][0]["content"][0], future_extension); +} + +#[test] +fn malformed_provider_extension_blocks_are_not_sent() { + let request = ModelRequest::new(vec![Message::User(crate::message::UserMessage { + content: vec![ContentBlock::ProviderExtension(json!({"opaque": true}))], + })]); + assert!( + request_body(&request, "m")["messages"] + .as_array() + .unwrap() + .is_empty() + ); + + let native_block = ModelRequest::new(vec![Message::User(crate::message::UserMessage { + content: vec![ContentBlock::ProviderExtension(json!({"type": "text"}))], + })]); + assert!( + request_body(&native_block, "m")["messages"] + .as_array() + .unwrap() + .is_empty() + ); } #[test] @@ -574,10 +640,17 @@ fn temperature_policy_uses_the_effective_request_model() { #[test] fn debug_redacts_the_api_key() { - let model = AnthropicModel::new("secret-api-key"); + let model = + AnthropicModel::new("secret-api-key").with_header("anthropic-beta", "secret-beta-value"); let debug = format!("{model:?}"); assert!(debug.contains("[redacted]")); assert!(!debug.contains("secret-api-key")); + assert!(!debug.contains("secret-beta-value")); + assert!(debug.contains("anthropic-beta")); + assert_eq!( + model.extra_headers, + vec![("anthropic-beta".into(), "secret-beta-value".into())] + ); } fn sse(events: &[serde_json::Value]) -> Vec { @@ -683,6 +756,74 @@ async fn streaming_thinking_arrives_on_the_reasoning_channel_with_its_signature( )); } +#[tokio::test] +async fn streaming_preserves_unknown_content_blocks_for_the_host() { + let extension = json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": { "query": "rust" } + }); + let events = [ + json!({"type":"message_start","message":{"id":"m","usage":{"input_tokens":1,"output_tokens":0}}}), + json!({"type":"content_block_start","index":0,"content_block":extension.clone()}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}), + json!({"type":"message_stop"}), + ]; + let items: Vec = stream::stream_from_bytes(vec![sse(&events)], "m") + .collect() + .await; + let Some(ModelStreamItem::Completed(response)) = items.last() else { + panic!("stream must end in Completed, got {:?}", items.last()); + }; + assert_eq!( + response.message.content, + vec![ContentBlock::ProviderExtension(extension)] + ); +} + +#[tokio::test] +async fn streaming_reconstructs_extension_input_fragments_and_boundaries() { + let events = [ + json!({"type":"message_start","message":{"id":"m","usage":{"input_tokens":1,"output_tokens":0}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{}}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"query\":"}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"rust\"}"}}), + json!({"type":"content_block_stop","index":0}), + json!({"type":"message_stop"}), + ]; + let items: Vec = stream::stream_from_bytes(vec![sse(&events)], "m") + .collect() + .await; + + assert!(items.iter().any(|item| matches!( + item, + ModelStreamItem::BlockStart { + index: 0, + kind: BlockKind::ProviderExtension { block_type } + } if block_type == "server_tool_use" + ))); + assert!(items.iter().any(|item| matches!( + item, + ModelStreamItem::BlockEnd { + index: 0, + block: ContentBlock::ProviderExtension(value) + } if value["input"] == json!({"query": "rust"}) + ))); + assert!(matches!( + items.last(), + Some(ModelStreamItem::Completed(response)) + if response.message.content + == vec![ContentBlock::ProviderExtension(json!({ + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "rust"} + }))] + )); + assert!(matches!(items.last(), Some(ModelStreamItem::Completed(_)))); +} + #[tokio::test] async fn streaming_without_message_stop_is_a_provider_failure() { let events = [ diff --git a/crates/tinyinference-llm/src/providers/openai/test.rs b/crates/tinyinference-llm/src/providers/openai/test.rs index 45aeb7a..93013d2 100644 --- a/crates/tinyinference-llm/src/providers/openai/test.rs +++ b/crates/tinyinference-llm/src/providers/openai/test.rs @@ -10,8 +10,8 @@ use serde_json::json; use super::*; use crate::message::{ContentBlock, Message}; use crate::model::{ - BlockDelta, BlockKind, ChatModel, ModelRequest, ModelStreamItem, ProviderError, ResponseFormat, - StreamAccumulator, ToolChoice, + BlockDelta, BlockKind, ChatModel, ModelRequest, ModelStreamItem, ProviderError, + ReasoningEffort, ResponseFormat, StreamAccumulator, ToolChoice, }; use crate::providers::{ProviderKind, ProviderSpec}; use crate::tool::ToolSchema; @@ -109,6 +109,16 @@ fn translates_request_to_openai_json_shape() { assert_eq!(value["seed"], json!(7)); } +#[test] +fn translates_maximum_reasoning_effort() { + let request = ModelRequest::new(vec![Message::user("solve")]) + .with_model("o3") + .with_reasoning_effort(ReasoningEffort::XHigh); + let body = model().translate_request(&request).unwrap(); + let value = serde_json::to_value(body).unwrap(); + assert_eq!(value["reasoning_effort"], "xhigh"); +} + #[test] fn custom_messages_are_never_sent_to_the_provider() { let request = ModelRequest::new(vec![