From d9c819f599322108d8debd0df16608a7e7cb3547 Mon Sep 17 00:00:00 2001 From: ChethanUK Date: Wed, 30 Sep 2026 14:40:20 +0200 Subject: [PATCH] feat(routing): support json_object output for custom classifiers Signed-off-by: ChethanUK --- crates/libsy/src/algorithms/llm_class.rs | 10 +- .../algorithms/util/classifier_contract.rs | 212 +++++++++++++++--- crates/switchyard-py/src/libsy_bindings.rs | 26 ++- crates/switchyard-runner/src/algorithm.rs | 7 +- crates/switchyard-runner/src/config.rs | 29 +++ crates/switchyard-server/tests/server.rs | 179 ++++++++++++++- docs/reference/toml_schema.md | 2 +- .../llm_classifier_routing.md | 19 +- switchyard_rust/libsy.py | 1 + tests/test_libsy_minimal_bindings.py | 74 ++++++ 10 files changed, 499 insertions(+), 60 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 488bc6585..06e5449cc 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -454,7 +454,8 @@ impl CustomClassifierPolicy { pub struct CustomClassifierConfig { /// System prompt sent to the classifier judge. pub prompt: String, - /// Inner JSON Schema placed inside the provider's structured-output wrapper. + /// Inner JSON Schema for the verdict. Sent in the provider's structured-output wrapper in + /// JSON Schema mode, or appended to the prompt in JSON Object mode. pub response_schema: Value, /// Deterministic policy applied after the verdict passes schema validation. pub policy: CustomClassifierPolicy, @@ -466,6 +467,8 @@ pub struct CustomClassifierConfig { pub recent_turn_window: Option, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, + /// Structured-output mode requested from the classifier judge. + pub response_format_type: ClassifierResponseFormat, } impl CustomClassifierConfig { @@ -483,6 +486,7 @@ impl CustomClassifierConfig { message_hash_fallback: false, recent_turn_window: None, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, + response_format_type: ClassifierResponseFormat::default(), } } @@ -666,8 +670,10 @@ impl LlmTaskClassifier { message_hash_fallback, recent_turn_window, max_output_tokens, + response_format_type, } = config; - let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; + let contract = + ClassifierContract::from_inner_schema(&prompt, response_schema, response_format_type)?; let policy = match policy { CustomClassifierPolicy::TargetSelector { selector } => { CustomPolicyRuntime::TargetSelector(TargetSelectorPolicy::new(selector)?) diff --git a/crates/libsy/src/algorithms/util/classifier_contract.rs b/crates/libsy/src/algorithms/util/classifier_contract.rs index 6cdc42551..defe4b53d 100644 --- a/crates/libsy/src/algorithms/util/classifier_contract.rs +++ b/crates/libsy/src/algorithms/util/classifier_contract.rs @@ -78,6 +78,7 @@ impl ClassifierContract { response_format_json: &str, ) -> Result { let prompt_template = config.prompt().unwrap_or(default_prompt); + validate_prompt(prompt_template)?; let response_format: Value = serde_json::from_str(response_format_json).map_err(|error| { LibsyError::AlgorithmError { @@ -90,29 +91,33 @@ impl ClassifierContract { message: "response schema has no json_schema.schema".to_string(), })?; match config.response_format_type() { - ClassifierResponseFormat::JsonSchema => { - Self::from_response_format(prompt_template, response_format, None) - } + ClassifierResponseFormat::JsonSchema => Ok(Self::from_response_format( + prompt_template.to_string(), + response_format, + None, + )), ClassifierResponseFormat::JsonObject => { - validate_prompt(prompt_template)?; let validator = compile_schema(schema)?; - let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| { - algorithm_error(format!("response schema could not be rendered: {error}")) - })?; - let system_prompt = format!( - "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}" - ); - Self::from_response_format( - &system_prompt, + Ok(Self::from_response_format( + schema_in_prompt(prompt_template, schema)?, json!({"type": "json_object"}), Some(validator), - ) + )) } } } - /// Builds a provider response format around a user-supplied inner JSON Schema. - pub(crate) fn from_inner_schema(prompt_template: &str, schema: Value) -> Result { + /// Builds a contract around a user-supplied inner JSON Schema. + /// + /// The verdict is validated against the schema locally in both modes. JSON Schema mode sends it through the + /// provider's strict wrapper; JSON Object mode appends it to the prompt, because the + /// provider then guarantees only that the reply is JSON. + pub(crate) fn from_inner_schema( + prompt_template: &str, + schema: Value, + response_format_type: ClassifierResponseFormat, + ) -> Result { + validate_prompt(prompt_template)?; if schema.get("json_schema").is_some() { return Err(LibsyError::AlgorithmError { message: @@ -121,32 +126,53 @@ impl ClassifierContract { }); } let validator = compile_schema(&schema)?; - Self::from_response_format( - prompt_template, - json!({ - "type": "json_schema", - "json_schema": { - "name": "switchyard_classifier_response", - "strict": true, - "schema": schema, + match response_format_type { + ClassifierResponseFormat::JsonSchema => Ok(Self::from_response_format( + prompt_template.to_string(), + json!({ + "type": "json_schema", + "json_schema": { + "name": "switchyard_classifier_response", + "strict": true, + "schema": schema, + } + }), + Some(validator), + )), + ClassifierResponseFormat::JsonObject => { + // Catches the common case of a root `type` that excludes objects: the provider + // only returns objects, so every verdict would fail and route to the default. + // A schema with no `type` (e.g. `{"enum": [1, 2]}`) is not checked and can + // still never match; that is left to the author. + let is_object_allowed = match schema.get("type") { + Some(Value::String(root)) => root == "object", + Some(Value::Array(roots)) => roots.iter().any(|root| root == "object"), + _ => true, + }; + if !is_object_allowed { + return Err(algorithm_error( + "response_schema must describe a JSON object in json_object mode", + )); } - }), - Some(validator), - ) + Ok(Self::from_response_format( + schema_in_prompt(prompt_template, &schema)?, + json!({"type": "json_object"}), + Some(validator), + )) + } + } } fn from_response_format( - prompt_template: &str, + system_prompt: String, response_format: Value, validator: Option, - ) -> Result { - validate_prompt(prompt_template)?; - - Ok(Self { - system_prompt: prompt_template.to_string(), + ) -> Self { + Self { + system_prompt, response_format, validator, - }) + } } pub(crate) fn system_prompt(&self) -> &str { @@ -187,6 +213,19 @@ pub(super) fn validate_prompt(prompt_template: &str) -> Result<()> { Ok(()) } +/// Appends the verdict schema to a prompt for JSON Object mode. +/// +/// Callers validate the template, not this result: the appended schema makes the prompt +/// non-empty and may itself contain `{{RESPONSE_SCHEMA}}` text. +fn schema_in_prompt(prompt_template: &str, schema: &Value) -> Result { + let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| { + algorithm_error(format!("response schema could not be rendered: {error}")) + })?; + Ok(format!( + "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}" + )) +} + fn compile_schema(schema: &Value) -> Result { if !schema.is_object() { return Err(algorithm_error("response_schema must be a JSON object")); @@ -307,6 +346,7 @@ mod tests { "required": ["decision"], "additionalProperties": false }), + ClassifierResponseFormat::JsonSchema, )?; assert_eq!( @@ -335,12 +375,110 @@ mod tests { #[test] fn a_provider_wrapper_is_rejected_as_an_inner_schema() { + for response_format_type in [ + ClassifierResponseFormat::JsonSchema, + ClassifierResponseFormat::JsonObject, + ] { + let error = ClassifierContract::from_inner_schema( + "classify", + json!({"json_schema": {"schema": {"type": "object"}}}), + response_format_type, + ) + .expect_err("a wrapped schema should error"); + + assert!( + error.to_string().contains("inner JSON Schema"), + "{response_format_type:?}: {error}" + ); + } + } + + #[test] + fn a_custom_contract_can_request_a_json_object() -> Result<()> { + let contract = ClassifierContract::from_inner_schema( + "Choose a target.", + json!({ + "type": "object", + "properties": {"target": {"type": "string", "enum": ["sonnet", "opus"]}}, + "required": ["target"], + "additionalProperties": false + }), + ClassifierResponseFormat::JsonObject, + )?; + + assert_eq!(contract.response_format(), &json!({"type": "json_object"})); + assert!(contract.system_prompt().starts_with("Choose a target.")); + assert!(contract.system_prompt().contains("JSON Schema")); + assert!(contract.system_prompt().contains("\"target\"")); + // The provider no longer enforces the schema, so the local validator must. + contract.validate_verdict(&json!({"target": "sonnet"}))?; + assert!( + contract + .validate_verdict(&json!({"target": "unknown"})) + .is_err() + ); + Ok(()) + } + + #[test] + fn json_object_mode_accepts_only_schemas_whose_root_allows_an_object() { + for (schema, accepted) in [ + (json!({"type": "object"}), true), + (json!({"type": ["object"]}), true), + (json!({"type": ["object", "null"]}), true), + (json!({"type": "string"}), false), + (json!({"type": "array"}), false), + (json!({"type": ["string", "null"]}), false), + ] { + let result = ClassifierContract::from_inner_schema( + "classify", + schema.clone(), + ClassifierResponseFormat::JsonObject, + ); + + match result { + Ok(_) => assert!(accepted, "{schema} should be rejected"), + Err(error) => { + assert!(!accepted, "{schema}: {error}"); + assert!( + error.to_string().contains("must describe a JSON object"), + "{schema}: {error}" + ); + } + } + } + } + + #[test] + fn a_schema_that_mentions_the_placeholder_is_accepted_in_both_modes() { + let schema = json!({ + "type": "object", + "properties": {"target": {"type": "string"}}, + "required": ["target"], + "description": "{{RESPONSE_SCHEMA}}" + }); + for response_format_type in [ + ClassifierResponseFormat::JsonSchema, + ClassifierResponseFormat::JsonObject, + ] { + ClassifierContract::from_inner_schema("pick", schema.clone(), response_format_type) + .unwrap_or_else(|error| panic!("{response_format_type:?}: {error}")); + } + } + + #[test] + fn an_empty_prompt_is_rejected_in_json_object_mode() { let error = ClassifierContract::from_inner_schema( - "classify", - json!({"json_schema": {"schema": {"type": "object"}}}), + " ", + json!({"type": "object"}), + ClassifierResponseFormat::JsonObject, ) - .expect_err("provider wrapper should be rejected"); + .expect_err("an empty prompt should be rejected before the schema is appended"); - assert!(error.to_string().contains("inner JSON Schema")); + assert!( + error + .to_string() + .contains("classifier prompt must not be empty") + ); } } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 23c90d7ad..b049c5429 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -218,7 +218,8 @@ impl PyCustomClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, - max_output_tokens=4096 + max_output_tokens=4096, + response_format_type="json_schema" ))] #[allow(clippy::too_many_arguments)] fn new( @@ -229,6 +230,7 @@ impl PyCustomClassifierConfig { message_hash_fallback: bool, recent_turn_window: Option, max_output_tokens: u64, + response_format_type: &str, ) -> PyResult { // Convert the Python schema into serde JSON and pair it with the target-selector policy; // conversion failures propagate to Python through `PyResult`. @@ -241,6 +243,7 @@ impl PyCustomClassifierConfig { inner.message_hash_fallback = message_hash_fallback; inner.recent_turn_window = recent_turn_window; inner.max_output_tokens = max_output_tokens; + inner.response_format_type = parse_response_format_type(response_format_type)?; Ok(Self { inner }) } } @@ -348,16 +351,17 @@ fn classifier_contract( if let Some(prompt) = prompt { contract = contract.with_prompt(prompt); } - let response_format_type = match response_format_type { - "json_schema" => ClassifierResponseFormat::JsonSchema, - "json_object" => ClassifierResponseFormat::JsonObject, - other => { - return Err(PyValueError::new_err(format!( - "response_format_type must be 'json_schema' or 'json_object', got {other:?}" - ))); - } - }; - Ok(contract.with_response_format_type(response_format_type)) + Ok(contract.with_response_format_type(parse_response_format_type(response_format_type)?)) +} + +fn parse_response_format_type(response_format_type: &str) -> PyResult { + match response_format_type { + "json_schema" => Ok(ClassifierResponseFormat::JsonSchema), + "json_object" => Ok(ClassifierResponseFormat::JsonObject), + other => Err(PyValueError::new_err(format!( + "response_format_type must be 'json_schema' or 'json_object', got {other:?}" + ))), + } } /// Judge target and policy used when stage-router signals are inconclusive. diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index e742d48ba..187030a48 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -133,6 +133,7 @@ struct CustomClassifierRouteConfig { message_hash_fallback: bool, recent_turn_window: Option, max_output_tokens: u64, + response_format_type: ClassifierResponseFormat, } /// Runtime model groups for a custom classifier, keyed by group name. @@ -993,10 +994,9 @@ impl LlmClassifierRouteConfig { || base_threshold.is_some() || threshold_step.is_some() || escalation.is_some() - || *response_format_type != ClassifierResponseFormat::JsonSchema { return Err(AlgorithmConfigError::new(format!( - "llm_classifier route {route_name} mode custom cannot use capability or escalation fields and response_format_type must be 'json_schema'" + "llm_classifier route {route_name} mode custom cannot use capability or escalation fields" ))); } let models = required_classifier_field(route_name, "models", models)?; @@ -1032,6 +1032,7 @@ impl LlmClassifierRouteConfig { message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, max_output_tokens: *max_output_tokens, + response_format_type: *response_format_type, }, )) } @@ -1118,6 +1119,7 @@ fn build_subagent_router_config( ); classifier_config.recent_turn_window = config.recent_turn_window; classifier_config.max_output_tokens = config.max_output_tokens; + classifier_config.response_format_type = config.response_format_type; let classifier = Arc::new( LlmTaskClassifier::new(LlmClassifierConfig::Custom { default_target: config.default_target.clone(), @@ -1271,6 +1273,7 @@ fn build_algorithm( classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; classifier_config.max_output_tokens = config.max_output_tokens; + classifier_config.response_format_type = config.response_format_type; LlmTaskClassifier::new(LlmClassifierConfig::Custom { default_target: config.default_target, config: classifier_config, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index f37b4cda9..2a9e76753 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1289,6 +1289,35 @@ new = ["send_message"] ); } + #[test] + fn mode_custom_accepts_json_object_output() -> RunnerResult<()> { + let top_level = format!( + r#"{VALID_CONFIG} +[routes.custom] +id = "switchyard/custom" +type = "llm_classifier" +mode = "custom" +response_format_type = "json_object" +models = {{ judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }} +default_target = "efficient" +prompt = "Select a target for this task." +response_schema = '{{"type":"object","properties":{{"target":{{"type":"string","enum":["capable","efficient"]}}}},"required":["target"],"additionalProperties":false}}' +policy = {{ type = "target_selector", selector = "/target" }} +"# + ); + let subagent = with_subagent_llm_classifier( + VALID_CONFIG, + "passthrough", + "\nresponse_format_type = \"json_object\"", + ); + + for (name, config) in [("top-level", top_level), ("subagent", subagent)] { + runner_from_toml(&config) + .map_err(|error| RunnerError::configuration(format!("{name}: {error}")))?; + } + Ok(()) + } + #[test] fn stage_router_rejects_an_unknown_field() { let config = stage_config().replace( diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index cd21d3c91..d8257a2d2 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -371,9 +371,15 @@ async fn upstream_chat( .into_response(); } + // JSON Object mode carries the custom schema in the judge prompt instead of + // `response_format`, so recognize it there too. let custom_target_schema = body .pointer("/response_format/json_schema/schema/properties/decision/properties/target") - .is_some(); + .is_some() + || (body["response_format"]["type"] == "json_object" + && body["messages"][0]["content"] + .as_str() + .is_some_and(|prompt| prompt.contains("\"decision\""))); let requests_invalid_verdict = body["messages"].as_array().is_some_and(|messages| { messages.iter().any(|message| { message["content"] @@ -399,7 +405,10 @@ async fn upstream_chat( }) }); let content = if model == "model/classifier" && custom_target_schema { - if requests_invalid_verdict { + if requests_schema_invalid_verdict { + // The selector would accept "fast"; only the schema's additionalProperties rejects it. + r#"{"decision":{"target":"fast"},"extra":1}"#.to_string() + } else if requests_invalid_verdict { r#"{"decision":{"target":"unknown"}}"#.to_string() } else { let group = requested_group.unwrap_or_else(|| "efficient".to_string()); @@ -2486,6 +2495,172 @@ selector = "/decision/target" Ok(()) } +#[tokio::test] +async fn custom_classifier_can_request_json_object_output() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.custom] +id = "switchyard/custom" +type = "llm_classifier" +mode = "custom" +response_format_type = "json_object" +models = {{ judge = ["classifier"], fast = ["weak"], reasoning = ["strong"], any = ["weak", "strong"] }} +default_target = "reasoning" +prompt = "CUSTOM JSON OBJECT" +response_schema = ''' +{{ + "type": "object", + "properties": {{ + "decision": {{ + "type": "object", + "properties": {{ + "target": {{"type": "string", "enum": ["fast", "reasoning"]}} + }}, + "required": ["target"], + "additionalProperties": false + }} + }}, + "required": ["decision"], + "additionalProperties": false +}} +''' + +[routes.custom.policy] +type = "target_selector" +selector = "/decision/target" +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + // The provider only guarantees JSON here. The second verdict names a valid target, so the + // selector alone would pick `fast`; only local schema validation rejects it and falls + // back to `default_target`. + for (task, selected) in [ + ("route to fast", "model/weak"), + ("return a schema-invalid verdict", "model/strong"), + ] { + upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/custom", + "messages": [{"role": "user", "content": task}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK, "{task}"); + assert_eq!( + response.headers["x-model-router-selected-model"], selected, + "{task}" + ); + + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("custom classifier target was not called")?; + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}) + ); + let prompt = judge_call["messages"][0]["content"] + .as_str() + .ok_or("custom classifier prompt was not text")?; + assert!(prompt.starts_with("CUSTOM JSON OBJECT"), "{prompt}"); + assert!(prompt.contains("JSON Schema"), "{prompt}"); + assert!(prompt.contains("\"decision\""), "{prompt}"); + } + Ok(()) +} + +#[tokio::test] +async fn subagent_custom_classifier_can_request_json_object_output() -> TestResult { + let upstream = MockUpstream::start().await?; + let app = build_switchyard_router(load_test_config(&format!( + r#" +schema_version = 1 +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" +[targets] +classifier = {{ id = "model/classifier", llm_client = "upstream" }} +strong = {{ id = "model/strong", llm_client = "upstream" }} +weak = {{ id = "model/weak", llm_client = "upstream" }} +[routes.agent] +id = "agent" +type = "passthrough" +target = "weak" +[routes.agent.subagents] +type = "llm_classifier" +mode = "custom" +response_format_type = "json_object" +models = {{ judge = ["classifier"], capable = ["strong"], efficient = ["weak"], any = ["strong", "weak"] }} +default_target = "efficient" +prompt = "classify the delegated task" +response_schema = '''{{"type":"object","properties":{{"decision":{{"type":"object","properties":{{"target":{{"type":"string","enum":["capable","efficient"]}}}},"required":["target"],"additionalProperties":false}}}},"required":["decision"],"additionalProperties":false}}''' +[routes.agent.subagents.policy] +type = "target_selector" +selector = "/decision/target" +"#, + base_url = upstream.base_url + ))?); + + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "agent", + "messages": [{"role": "user", "content": "route to capable"}] + })), + &[ + ("x-claude-code-session-id", "root-session"), + ("x-claude-code-agent-id", "child-agent"), + ], + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.headers["x-model-router-selected-model"], + "model/strong" + ); + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("subagent classifier target was not called")?; + // The sub-agent path builds its own classifier config; this pins that the + // setting reaches the judge instead of silently staying on JSON Schema. + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}) + ); + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 8bf382a08..a5b524691 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -229,7 +229,7 @@ Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Capability, escalation | — | Target the judge is called through. Not a routing destination. Custom mode uses `models.judge`. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | -| `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | +| `response_format_type` | No | `json_schema` | Structured-output mode for the judge in every mode. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode appends `response_schema`; do not copy it into the prompt. | Capability mode classifies before serving. See [LLM Classifier Routing](../routing_algorithms/llm_classifier_routing.md). diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index de35015ff..537ca426d 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -92,7 +92,7 @@ Switchyard does not parse provider-specific reasoning fields such as to `strong_target` even when the judge request returned HTTP 200. With session affinity, that fallback can be reused without another judge call. -Capability and escalation routes use JSON Schema structured output by default. +Every classifier mode uses JSON Schema structured output by default. For a provider that supports JSON Object mode but not JSON Schema, set `response_format_type = "json_object"` on the route. Switchyard then adds the verdict schema to the judge prompt and validates the returned object locally. @@ -123,7 +123,7 @@ for the server merge behavior. | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | -| `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. | +| `response_format_type` | `json_schema` | Structured-output mode for the judge in every mode. Use `json_object` for providers without JSON Schema support. | | `max_output_tokens` | `4096` | Maximum completion tokens available to the classifier verdict. Must be at least `1`. | ### Override the classifier prompt @@ -200,9 +200,11 @@ type = "target_selector" selector = "/decision/target" ``` -The names in `models` reference existing target tables. Switchyard passes the -schema to the provider in a strict structured-output wrapper and validates the -returned JSON again. `jsonptr` resolves the selector against that verdict. A +The names in `models` reference existing target tables. By default Switchyard +passes the schema to the provider in a strict structured-output wrapper; with +`response_format_type = "json_object"` it appends the schema to the prompt +instead (see below). Either way it validates the returned JSON against the +schema. `jsonptr` resolves the selector against that verdict. A missing, non-string, or unconfigured label falls back to `default_target`, and `judge` is never routable. @@ -218,6 +220,13 @@ order and is not a completion destination. carry the tier meaning the stage and composite routers give it; otherwise any name works. +If the judge's provider supports JSON Object mode but not JSON Schema, add +`response_format_type = "json_object"` to the route. `response_schema` is still +required. Switchyard appends it to your prompt, asks for a JSON object, and +validates the reply against it. The configured schema is the source of truth, so +do not paste a copy into the prompt. A reply that fails the schema falls back to +`default_target`. + This separation applies to every classifier mode. Prompts containing the legacy `{{RESPONSE_SCHEMA}}` placeholder are rejected during configuration validation. diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 7bdf2cb6c..491bc4e7a 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -77,6 +77,7 @@ def __init__( message_hash_fallback: bool = False, recent_turn_window: int | None = None, max_output_tokens: int = 4096, + response_format_type: Literal["json_schema", "json_object"] = "json_schema", ) -> None: ... @final diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index e28ffc126..d9badd2ae 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -358,6 +358,64 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +async def test_custom_classifier_config_accepts_json_object_output() -> None: + """Verify that Python can select JSON Object mode for a custom classifier judge.""" + + class JudgeClient(EchoClient): + async def call(self, request: dict[str, Any]) -> dict[str, Any]: + self.calls.append(request) + return { + "model": self.model, + "outputs": [ + { + "role": "assistant", + "content": [{"type": "text", "text": '{"target":"efficient"}'}], + "stop_reason": "end_turn", + } + ], + } + + schema = { + "type": "object", + "additionalProperties": False, + "required": ["target"], + "properties": {"target": {"type": "string", "enum": ["capable", "efficient"]}}, + } + judge = JudgeClient("judge") + algorithm = algorithms.llm_classifier( + LlmClassifierConfig.custom( + default_target="capable", + config=CustomClassifierConfig( + "Choose a target.", + schema, + "/target", + response_format_type="json_object", + ), + ) + ) + + _, response = await run_algorithm( + algorithm, + { + "judge": judge, + "model-a": EchoClient("model-a"), + "model-b": EchoClient("model-b"), + }, + models={ + "judge": ["judge"], + "capable": ["model-a"], + "efficient": ["model-b"], + "any": ["model-a", "model-b"], + }, + ) + + assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"} + prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] + assert prompt.startswith("Choose a target.") + assert '"efficient"' in prompt + assert response["model"] == "model-b" + + def test_classifier_config_rejects_unknown_response_format() -> None: invalid_response_format: Any = "yaml" @@ -368,6 +426,22 @@ def test_classifier_config_rejects_unknown_response_format() -> None: TaskClassifierConfig(0.5, response_format_type=invalid_response_format) +def test_custom_classifier_config_rejects_unknown_response_format() -> None: + invalid_response_format: Any = "yaml" + schema = {"type": "object"} + + with pytest.raises( + ValueError, + match="response_format_type must be 'json_schema' or 'json_object'", + ): + CustomClassifierConfig( + "Choose a target.", + schema, + "/target", + response_format_type=invalid_response_format, + ) + + async def test_random_weights_and_seed_are_reproducible() -> None: def algorithm(): return algorithms.random(