diff --git a/Cargo.lock b/Cargo.lock index b1cbce1be..bcd6c200a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2358,6 +2358,7 @@ dependencies = [ "opentelemetry_sdk", "parking_lot", "reqwest", + "serde", "serde_json", "switchyard-libsy", "switchyard-protocol", diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml index 9a9ca6e0a..a347ed7cc 100644 --- a/crates/libsy-llm-client/Cargo.toml +++ b/crates/libsy-llm-client/Cargo.toml @@ -29,6 +29,7 @@ parking_lot.workspace = true http.workspace = true httpdate.workspace = true serde_json.workspace = true +serde.workspace = true tokio.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 34ec8a989..160a428e1 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -252,6 +252,10 @@ fn build_multi_format_client( transport failures, timeouts, HTTP 408/429, and 5xx responses. Buffered body transport failures are retried; streaming body failures are not replayed after the response has been returned. +- `ModelConfig::with_responses_reasoning` controls Responses reasoning replay. + Every Responses model defaults to `PreserveEncrypted`: plaintext is removed, + encrypted provider state is retained. Set `Drop` explicitly for a backend + that cannot consume encrypted reasoning. Messages and tool history are retained. - `HttpBackendConfig::timeout` bounds one complete response, including retries, retry delays, and every stream read. Expiry returns `LlmClientError::Timeout`, either from the call or from the returned stream, which then ends. `None` leaves diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index f2d0301dd..8521395bf 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -63,6 +63,7 @@ pub struct ModelConfig { model_name: ModelId, default_backend: Backend, other_backends: Option>, + responses_reasoning: crate::ResponsesReasoningPolicy, } impl ModelConfig { @@ -77,8 +78,16 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + responses_reasoning: crate::ResponsesReasoningPolicy::default(), } } + + /// Sets how Responses reasoning items are replayed to this model. + #[must_use] + pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { + self.responses_reasoning = policy; + self + } } /// A model-bearing provider operation outside the normal completion endpoint. @@ -262,6 +271,13 @@ impl TranslatingLlmClient { } omit_configured_body_fields(&mut body, backend.omit_body_fields()); merge_extra_body(&mut body, backend.extra_body()); + if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_reasoning) + .unwrap_or_default() + .normalize(&mut body); + } // After the merge on purpose: the effort override must win over both the caller's // value and any `reasoning` default a target set through `extra_body`. apply_reasoning_effort(&mut body, backend); @@ -1446,6 +1462,13 @@ mod tests { )] } + fn local_responses_map(base_url: &str) -> Vec { + vec![ + ModelConfig::new("local", Backend::OpenAiResponses(config(base_url)), None) + .with_responses_reasoning(crate::ResponsesReasoningPolicy::Drop), + ] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -2192,6 +2215,203 @@ mod tests { Ok(()) } + #[tokio::test] + async fn responses_policy_normalizes_input_replaced_by_target_defaults() + -> std::result::Result<(), Box> { + for (policy, expected_reasoning) in [ + (crate::ResponsesReasoningPolicy::PreserveEncrypted, 1), + (crate::ResponsesReasoningPolicy::Drop, 0), + ] { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", "object": "response", "model": "gpt", + "status": "completed", "output": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .expect(1) + .mount(&server) + .await; + let mut backend = config(&format!("{}/v1", server.uri())); + backend.omit_body_fields.insert("input".to_string()); + backend.extra_body.insert("input".to_string(), json!([ + {"type": "message", "role": "user", "content": "replacement"}, + {"type": "reasoning", "encrypted_content": "opaque", "content": [{"type": "reasoning_text", "text": "private"}]}, + {"type": "reasoning", "encrypted_content": null, "content": [{"type": "reasoning_text", "text": "local"}]} + ])); + let client = TranslatingLlmClient::new(&[ModelConfig::new( + "gpt", + Backend::OpenAiResponses(backend), + None, + ) + .with_responses_reasoning(policy)])?; + client + .call_rewrite_model_raw( + json!({"model": "gpt", "input": "original"}), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + let requests = server.received_requests().await.expect("recorded requests"); + let body: Value = serde_json::from_slice(&requests[0].body)?; + let input = body["input"].as_array().expect("replacement input"); + assert_eq!(input[0]["content"], "replacement"); + let reasoning: Vec<_> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), expected_reasoning); + for item in reasoning { + assert_eq!(item["encrypted_content"], "opaque"); + assert_eq!(item["content"], json!([])); + } + } + Ok(()) + } + + // Strict Responses upstreams retain encrypted state without replaying plaintext. + #[tokio::test] + async fn responses_requests_drop_unsigned_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) + .collect(); + reasoning.len() == 1 + && reasoning[0] + .get("encrypted_content") + .and_then(Value::as_str) + == Some("encrypted") + && reasoning[0].get("content") == Some(&json!([])) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "object": "response", + "model": "gpt", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": "inspect the repo"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "private local reasoning"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must not be replayed"}], + "encrypted_content": "encrypted" + } + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + + // Encrypted hosted reasoning is opaque to a local Responses backend. Dropping + // it avoids llama.cpp rejecting a missing or empty `content` array while + // retaining the conversation and tool-call history it can consume. + #[tokio::test] + async fn local_responses_requests_drop_encrypted_reasoning_items() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let Some(input) = body.get("input").and_then(Value::as_array) else { + return false; + }; + input + .iter() + .all(|item| item.get("type").and_then(Value::as_str) != Some("reasoning")) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call") + }) + && input.iter().any(|item| { + item.get("type").and_then(Value::as_str) == Some("function_call_output") + }) + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_local", + "object": "response", + "model": "local", + "status": "completed", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2} + }))) + .mount(&server) + .await; + + let client = + TranslatingLlmClient::new(&local_responses_map(&format!("{}/v1", server.uri())))?; + let raw = json!({ + "model": "client-facing", + "input": [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "inspect"}]}, + { + "type": "reasoning", + "encrypted_content": "opaque-provider-reasoning" + }, + {"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]} + ] + }); + + client + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("local")), + WireFormat::OpenAiResponses, + ) + .await?; + Ok(()) + } + // A router can serve earlier turns from an OpenAI target and later turns from // an Anthropic one, so the Anthropic leg must drop OpenAI-only fields the // caller keeps sending or the upstream rejects the whole request. diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index d5f579c7d..714f58063 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -23,6 +23,7 @@ pub mod metrics; mod observability; mod observation; pub mod raw; +mod responses_reasoning; pub mod run; pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig}; @@ -30,6 +31,7 @@ pub use client::{AuxiliaryOperation, ModelConfig, TranslatingLlmClient}; pub use error::{LlmClientError, Result}; pub use observation::{LlmCallObservation, RunObservation, RunObserver}; pub use raw::RawResponse; +pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, decide, run}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/responses_reasoning.rs b/crates/libsy-llm-client/src/responses_reasoning.rs new file mode 100644 index 000000000..2c0e21c30 --- /dev/null +++ b/crates/libsy-llm-client/src/responses_reasoning.rs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Target-specific replay policy for OpenAI Responses reasoning items. + +use serde::Deserialize; +use serde_json::Value; + +/// Controls which Responses reasoning items are replayed to an upstream. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesReasoningPolicy { + /// Preserve provider-encrypted reasoning but remove plaintext reasoning. + /// + /// This is the default for every Responses model, regardless of its URL. + #[default] + PreserveEncrypted, + /// Drop all reasoning items while preserving messages and tool-call history. + /// + /// Use this for local Responses-compatible servers that cannot consume + /// another provider's encrypted reasoning representation. + Drop, +} + +impl ResponsesReasoningPolicy { + /// Normalizes a Responses request body for this replay policy. + pub(crate) fn normalize(self, body: &mut Value) { + let Some(Value::Array(input)) = body.get_mut("input") else { + return; + }; + input.retain_mut(|item| self.normalize_item(item)); + } + + // Keep non-reasoning items; PreserveEncrypted retains only non-empty encrypted_content. + fn normalize_item(self, item: &mut Value) -> bool { + let Some(object) = item.as_object_mut() else { + return true; + }; + if object.get("type").and_then(Value::as_str) != Some("reasoning") { + return true; + } + + let signed = matches!( + object.get("encrypted_content").and_then(Value::as_str), + Some(encrypted_content) if !encrypted_content.is_empty() + ); + if self == Self::PreserveEncrypted && signed { + object.insert("content".to_string(), Value::Array(Vec::new())); + true + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn mixed_history() -> Value { + json!({ + "input": [ + {"type": "message", "role": "user", "content": []}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "plaintext"}], + "encrypted_content": "" + }, + {"type": "function_call", "call_id": "call_1"}, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "must be removed"}], + "encrypted_content": "encrypted" + } + ] + }) + } + + #[test] + fn preserve_encrypted_drops_unsigned_and_clears_plaintext() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + let reasoning: Vec<&Value> = input + .iter() + .filter(|item| item["type"] == "reasoning") + .collect(); + assert_eq!(reasoning.len(), 1); + assert_eq!(reasoning[0]["encrypted_content"], "encrypted"); + assert_eq!(reasoning[0]["content"], json!([])); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } + + #[test] + fn unsigned_forms_are_removed_and_non_reasoning_is_unchanged() { + let message = json!({"type": "message", "role": "user", "content": "continue"}); + let mut body = json!({"input": [ + {"type": "reasoning"}, + {"type": "reasoning", "encrypted_content": null}, + {"type": "reasoning", "encrypted_content": ""}, + message.clone() + ]}); + ResponsesReasoningPolicy::PreserveEncrypted.normalize(&mut body); + assert_eq!(body["input"], json!([message])); + } + + #[test] + fn drop_removes_all_reasoning_and_keeps_tool_history() { + let mut body = mixed_history(); + ResponsesReasoningPolicy::Drop.normalize(&mut body); + + let input = body["input"].as_array().expect("input array"); + assert!(input.iter().all(|item| item["type"] != "reasoning")); + assert!(input.iter().any(|item| item["type"] == "function_call")); + assert!( + input + .iter() + .any(|item| item["type"] == "function_call_output") + ); + } +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 05f03e2ce..b326d4cac 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - TranslatingLlmClient, + ResponsesReasoningPolicy, TranslatingLlmClient, }; use switchyard_protocol::{Category, ModelId, RoutedLlmClient, WireFormat}; @@ -281,6 +281,13 @@ impl DeploymentConfig { for (name, client_config) in &self.llm_clients { validate_value("llm client name", name)?; + if client_config.responses_reasoning.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiResponses) + { + return Err(RunnerError::configuration(format!( + "llm client {name} responses_reasoning is only valid for openai_responses" + ))); + } let backend = build_backend( name, client_config, @@ -319,17 +326,20 @@ impl DeploymentConfig { ))); } } - model_configs.push(ModelConfig::new( - target.id.clone(), - build_backend( - &target.llm_client, - client_config, - &target.extra_body, - &target.omit_body_fields, - target.reasoning_effort.clone(), - )?, - None, - )); + model_configs.push( + ModelConfig::new( + target.id.clone(), + build_backend( + &target.llm_client, + client_config, + &target.extra_body, + &target.omit_body_fields, + target.reasoning_effort.clone(), + )?, + None, + ) + .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()), + ); } let mut clients = BTreeMap::new(); @@ -578,6 +588,7 @@ struct LlmClientConfig { max_retries: u32, /// Deadline in milliseconds for all attempts and the complete response. Unset is unbounded. timeout_ms: Option, + responses_reasoning: Option, } #[derive(Debug, Deserialize)] @@ -1752,6 +1763,61 @@ confidence_threshold = 0.5 Ok(()) } + #[test] + fn responses_reasoning_defaults_and_accepts_drop() -> RunnerResult<()> { + let default: DeploymentConfig = toml::from_str(VALID_CONFIG).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = default + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!(responses.responses_reasoning, None); + + let configured = VALID_CONFIG.replacen( + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"", + "[llm_clients.responses]\nformat = \"openai_responses\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"", + 1, + ); + let config: DeploymentConfig = toml::from_str(&configured).map_err(|error| { + RunnerError::configuration(format!("failed to parse config: {error}")) + })?; + let responses = config + .llm_clients + .get("responses") + .ok_or_else(|| RunnerError::configuration("responses llm client is missing"))?; + assert_eq!( + responses.responses_reasoning, + Some(ResponsesReasoningPolicy::Drop) + ); + runner_from_toml(&configured)?; + Ok(()) + } + + #[test] + fn responses_reasoning_rejects_other_formats() { + let invalid = VALID_CONFIG.replacen( + "format = \"openai_chat\"", + "format = \"openai_chat\"\nresponses_reasoning = \"drop\"", + 1, + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + + #[test] + fn responses_reasoning_rejects_unreferenced_other_format() { + let invalid = format!( + "{VALID_CONFIG}\n[llm_clients.unused]\nformat = \"openai_chat\"\nbase_url = \"https://example.test/v1\"\nresponses_reasoning = \"drop\"\n" + ); + assert!( + error_message(&invalid) + .contains("responses_reasoning is only valid for openai_responses") + ); + } + #[test] fn rejects_headers_that_switchyard_sets() { let cases = [ diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91..e87b37c04 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -5245,3 +5245,52 @@ async fn upstream_headers_forward_on_streaming_responses() -> TestResult { ); Ok(()) } + +#[tokio::test] +async fn configured_responses_reasoning_policy_reaches_upstream() -> TestResult { + let upstream = MockUpstream::start().await?; + let base_url = upstream.base_url.replace("/v1", "/buffered"); + for (setting, expected_reasoning) in [("", 1), ("responses_reasoning = \"drop\"", 0)] { + let config = format!( + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_responses" +base_url = "{base_url}" +{setting} +[targets] +answer = {{ id = "model/efficient", llm_client = "upstream" }} +[routes.route] +id = "route" +type = "passthrough" +target = "answer" +"# + ); + let app = build_switchyard_router(load_test_config(&config)?); + let response = send(&app, "POST", "/v1/responses", Some(json!({ + "model": "route", "input": [ + {"type": "message", "role": "user", "content": "continue"}, + {"type": "reasoning", "encrypted_content": "opaque", "summary": []}, + {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "local"}], "summary": []} + ] + }))).await?; + assert_eq!(response.status, StatusCode::OK, "{}", response.json()?); + let calls = upstream.calls.lock().await; + let input = calls.last().expect("upstream request")["input"] + .as_array() + .expect("input array"); + assert_eq!( + input + .iter() + .filter(|item| item["type"] == "reasoning") + .count(), + expected_reasoning + ); + assert!( + input + .iter() + .any(|item| item["type"] == "message" && item["role"] == "user") + ); + } + Ok(()) +} diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 7fc96b709..f0f420193 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -53,9 +53,18 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `api_key_env` | No | unset | Name of the environment variable holding the key. Omit to send no authentication. | | `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. All backends reachable through the route must use the same provider. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | +| `responses_reasoning` | No | `"preserve_encrypted"` | Responses reasoning replay: `"preserve_encrypted"` or `"drop"`. Only valid for `openai_responses`. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | | `timeout_ms` | No | unset | Deadline in milliseconds for all attempts, retry delays, and the complete response, including stream reads. Must be at least `1`. Unset leaves the wait unbounded. | +For `openai_responses`, `responses_reasoning` controls replay of reasoning +history. Every client defaults to `"preserve_encrypted"`: reasoning items with +non-empty `encrypted_content` are retained with empty `content`, and unsigned +reasoning is removed. Set `responses_reasoning = "drop"` explicitly for a backend +that cannot consume encrypted provider state. Both modes keep messages, tool +calls and tool results. The setting is rejected on other client formats, even +when no target uses the client. It is not inferred from the model name or URL. + The TOML never contains the secret itself. `api_key_env` names a variable that must exist and be non-empty when the server loads.