From 8e3917ee82f11574e34a4b3530a1546f73595245 Mon Sep 17 00:00:00 2001 From: Hiroshi Morishige Date: Wed, 23 Sep 2026 13:09:36 +0900 Subject: [PATCH 1/2] feat(libsy): make the unmatched verdict threshold step configurable Add `unmatched_steps` (0, 1 or 2, default 1) to capability-mode `llm_classifier` routes. An unmatched verdict has no capability rule behind its solve probability, so operators can require the unsupported-level threshold for it while uncertain verdicts keep one step. Stage, composite and Python bindings keep the default. Signed-off-by: Hiroshi Morishige --- crates/libsy/src/algorithms/llm_class.rs | 84 +++++++++++++++++-- crates/libsy/src/lib.rs | 4 +- crates/switchyard-py/src/libsy_bindings.rs | 10 ++- crates/switchyard-runner/src/algorithm.rs | 18 +++- crates/switchyard-server/README.md | 3 +- docs/reference/toml_schema.md | 3 +- .../llm_classifier_routing.md | 4 +- 7 files changed, 108 insertions(+), 18 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 488bc6585..1b78dafd2 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -73,10 +73,15 @@ impl TaskClassifierVerdict { } /// Returns the number of threshold steps assigned to this capability boundary. - fn boundary_steps(&self) -> Option { + /// + /// `unmatched_steps` comes from the route configuration: a request that matches no + /// capability rule has no evidence behind its solve probability, so operators can + /// require more confidence for it than for an uncertain verdict. + fn boundary_steps(&self, unmatched_steps: u8) -> Option { match self.capability_boundary.as_str() { "supported" => Some(0), - "uncertain" | "unmatched" => Some(1), + "uncertain" => Some(1), + "unmatched" => Some(unmatched_steps), "unsupported" => Some(2), _ => None, } @@ -220,6 +225,7 @@ impl ClassifierInput for TaskInput { struct TaskClassifierPolicy { base_threshold: f64, threshold_step: f64, + unmatched_steps: u8, } impl TaskClassifierPolicy { @@ -227,12 +233,14 @@ impl TaskClassifierPolicy { Self { base_threshold: config.base_threshold, threshold_step: config.threshold_step, + unmatched_steps: config.unmatched_steps, } } /// Returns the required solve probability for one validated verdict. fn threshold(&self, verdict: &TaskClassifierVerdict) -> Option { - Some(self.base_threshold + f64::from(verdict.boundary_steps()?) * self.threshold_step) + let steps = verdict.boundary_steps(self.unmatched_steps)?; + Some(self.base_threshold + f64::from(steps) * self.threshold_step) } } @@ -304,9 +312,13 @@ pub struct TaskClassifierConfig { pub base_threshold: f64, /// Amount added per capability-boundary step. /// - /// Supported verdicts use `base_threshold`, uncertain and unmatched verdicts use one - /// step, and unsupported verdicts use two steps. + /// Supported verdicts use `base_threshold`, uncertain verdicts use one step, unmatched + /// verdicts use `unmatched_steps`, and unsupported verdicts use two steps. pub threshold_step: f64, + /// Threshold steps applied to an unmatched verdict (no capability rule applies). + /// + /// Defaults to 1, the same as an uncertain verdict. 2 treats it like unsupported. + pub unmatched_steps: u8, /// How often the classifier re-decides this session's target. pub classify_trigger: ClassifyTrigger, /// Uses the first user message as the SessionKey for sticky routing when session metadata is unavailable. @@ -331,6 +343,8 @@ struct TaskClassifierConfigWire { base_threshold: f64, #[serde(default)] threshold_step: f64, + #[serde(default = "default_unmatched_steps")] + unmatched_steps: u8, #[serde(default)] classify_trigger: ClassifyTrigger, #[serde(default)] @@ -359,6 +373,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { Ok(Self { base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, + unmatched_steps: wire.unmatched_steps, classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, @@ -368,6 +383,13 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { } } +/// Unmatched verdicts share the uncertain step unless the route says otherwise. +pub const DEFAULT_UNMATCHED_STEPS: u8 = 1; + +const fn default_unmatched_steps() -> u8 { + DEFAULT_UNMATCHED_STEPS +} + const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } @@ -377,6 +399,7 @@ impl Default for TaskClassifierConfig { Self { base_threshold: 0.0, threshold_step: 0.0, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, @@ -405,6 +428,14 @@ impl TaskClassifierConfig { ), }); } + if self.unmatched_steps > 2 { + return Err(LibsyError::AlgorithmError { + message: format!( + "unmatched_steps must be 0, 1, or 2, got {}", + self.unmatched_steps + ), + }); + } let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step; if unsupported_threshold > 1.0 && unsupported_threshold - 1.0 > f64::EPSILON { return Err(LibsyError::AlgorithmError { @@ -1378,6 +1409,49 @@ mod tests { Ok(()) } + #[test] + fn unmatched_steps_raise_only_the_unmatched_threshold() -> Result<()> { + let policy = TaskClassifierPolicy::new(&TaskClassifierConfig { + threshold_step: 0.1, + unmatched_steps: 2, + ..test_config(0.75) + }); + + // 0.85 clears one step but not two: unmatched now routes to the capable target. + assert_eq!( + selected(&policy, Some(&verdict(0.85, "unmatched", "none")))?, + "capable" + ); + assert_eq!( + selected(&policy, Some(&verdict(0.95, "unmatched", "none")))?, + "efficient" + ); + // Other boundaries keep their steps. + assert_eq!( + selected(&policy, Some(&verdict(0.85, "uncertain", "UNC-1")))?, + "efficient" + ); + assert_eq!( + selected(&policy, Some(&verdict(0.75, "supported", "SUP-2")))?, + "efficient" + ); + Ok(()) + } + + #[test] + fn unmatched_steps_default_to_one_and_reject_values_above_two() { + let parsed: TaskClassifierConfig = + serde_json::from_value(serde_json::json!({ "base_threshold": 0.5 })) + .expect("config without unmatched_steps parses"); + assert_eq!(parsed.unmatched_steps, 1); + + let invalid = TaskClassifierConfig { + unmatched_steps: 3, + ..test_config(0.5) + }; + assert!(invalid.validate().is_err()); + } + /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index a1f1ef022..0cabc4b14 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -20,8 +20,8 @@ mod algorithms; pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger}; pub use algorithms::composite::{CompositeRouter, CompositeRouterConfig}; pub use algorithms::llm_class::{ - CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier, - TaskClassifierConfig, + CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, LlmClassifierConfig, + LlmTaskClassifier, TaskClassifierConfig, }; pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 23c90d7ad..31b003e10 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -14,10 +14,11 @@ use pyo3::prelude::*; use serde_json::Value; use switchyard_libsy::{ Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, - CustomClassifierConfig, CustomClassifierPolicy, DeescalationConfig, EscalationJudgeConfig, - HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, - LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter, - StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, + CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, DeescalationConfig, + EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, + LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, + StageRouter, StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, + ToolSemantics, }; use switchyard_protocol::{ Category, LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, @@ -330,6 +331,7 @@ impl PyTaskClassifierConfig { inner: TaskClassifierConfig { base_threshold, threshold_step, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index e742d48ba..214accfbc 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -12,10 +12,10 @@ use std::sync::Arc; use libsy::{ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, - CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, - PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, SubagentRouter, - SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, + CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, EscalationJudgeConfig, GateTrigger, + HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, + PickerMode, PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, + SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -103,6 +103,7 @@ struct CapabilityClassifierRouteConfig { weak_target: String, base_threshold: f64, threshold_step: f64, + unmatched_steps: u8, classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, @@ -237,6 +238,9 @@ pub struct LlmClassifierRouteConfig { /// Capability mode: how much to raise the threshold when the judge is /// uncertain. Added once for an uncertain verdict and twice for unsupported. pub threshold_step: Option, + /// Capability mode: threshold steps for an unmatched verdict (no capability rule + /// applies). 1 by default, the same as uncertain; 2 treats it like unsupported. + pub unmatched_steps: Option, /// How often the judge runs: every request, once per user turn, or once per session. pub classify_trigger: ClassifyTrigger, /// Reuses the session's target by hashing the first user message when no @@ -524,6 +528,7 @@ impl StageClassifierConfig { TaskClassifierConfig { base_threshold: self.base_threshold, threshold_step: self.threshold_step, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, @@ -878,6 +883,7 @@ impl LlmClassifierRouteConfig { weak_target, base_threshold, threshold_step, + unmatched_steps, classify_trigger, message_hash_fallback, recent_turn_window, @@ -933,6 +939,7 @@ impl LlmClassifierRouteConfig { base_threshold, )?, threshold_step: threshold_step.unwrap_or_default(), + unmatched_steps: unmatched_steps.unwrap_or(DEFAULT_UNMATCHED_STEPS), classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, @@ -959,6 +966,7 @@ impl LlmClassifierRouteConfig { if mode.is_some() && (base_threshold.is_some() || threshold_step.is_some() + || unmatched_steps.is_some() || *message_hash_fallback || recent_turn_window.is_some()) { @@ -992,6 +1000,7 @@ impl LlmClassifierRouteConfig { || weak_target.is_some() || base_threshold.is_some() || threshold_step.is_some() + || unmatched_steps.is_some() || escalation.is_some() || *response_format_type != ClassifierResponseFormat::JsonSchema { @@ -1231,6 +1240,7 @@ fn build_algorithm( let classifier_config = TaskClassifierConfig { base_threshold: config.base_threshold, threshold_step: config.threshold_step, + unmatched_steps: config.unmatched_steps, classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 152dbfe02..78b98cf4a 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -129,7 +129,8 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts | Key | Default | Meaning | |---|---|---| | `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. | -| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `unmatched_steps` | `1` | Threshold steps for an unmatched verdict. `0`, `1`, or `2`; `2` treats it like unsupported. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request including tool continuations, `user_turn` judges each new user message and holds that target across the tool calls between, `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `classify_trigger = "new_session"` or `"user_turn"`. | diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 8bf382a08..88c1ea0c8 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -239,7 +239,8 @@ Capability mode classifies before serving. See | `strong_target` | Yes | — | Capable tier. | | `weak_target` | Yes | — | Efficient tier. | | `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. | -| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. | | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index de35015ff..069a6139f 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -68,7 +68,8 @@ greater than or equal to the applicable threshold. Otherwise it routes to `strong_target`: - `supported` uses `base_threshold`. -- `uncertain` and `unmatched` use `base_threshold + threshold_step`. +- `uncertain` uses `base_threshold + threshold_step`. +- `unmatched` uses `base_threshold + unmatched_steps * threshold_step` (`unmatched_steps` defaults to `1`). - `unsupported` uses `base_threshold + 2 * threshold_step`. An invalid, inconsistent, or unparseable verdict routes to @@ -119,6 +120,7 @@ for the server merge behavior. |---|---|---| | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | +| `unmatched_steps` | `1` | Boundary steps applied to an `unmatched` verdict. `0`, `1`, or `2`; `2` requires the same confidence as an `unsupported` verdict. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. | From 64868bd0cce361b6428ee7a14fac8de6075e3553 Mon Sep 17 00:00:00 2001 From: Hiroshi Morishige Date: Fri, 25 Sep 2026 14:52:19 +0900 Subject: [PATCH 2/2] fix(runner): reject unmatched_steps on implicit escalation routes An escalation route selected by its `escalation` table alone tolerated `unmatched_steps` although escalation mode never reads it. The key is new, so no existing configuration depends on that, and the other capability keys keep their compatibility behavior in the implicit form. Adds a runner test covering acceptance, the value range, and the escalation rejection. Signed-off-by: Hiroshi Morishige --- crates/switchyard-runner/src/algorithm.rs | 15 +++++++----- crates/switchyard-runner/src/config.rs | 28 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index 214accfbc..f2d5c9276 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -963,12 +963,15 @@ impl LlmClassifierRouteConfig { "llm_classifier route {route_name} mode escalation cannot use classify_trigger" ))); } - if mode.is_some() - && (base_threshold.is_some() - || threshold_step.is_some() - || unmatched_steps.is_some() - || *message_hash_fallback - || recent_turn_window.is_some()) + // `unmatched_steps` is new, so no existing escalation configuration carries it: + // reject it even when the mode is implied by `escalation`. The older capability + // keys stay tolerated in that implicit form for compatibility. + if unmatched_steps.is_some() + || (mode.is_some() + && (base_threshold.is_some() + || threshold_step.is_some() + || *message_hash_fallback + || recent_turn_window.is_some())) { return Err(AlgorithmConfigError::new(format!( "llm_classifier route {route_name} mode escalation cannot use capability routing settings" diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index f37b4cda9..e732dd90d 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1183,6 +1183,34 @@ new = ["send_message"] Ok(()) } + #[test] + fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> { + // Accepted on a capability route; the value range is checked by the algorithm. + runner_from_toml(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 2", + ))?; + assert!( + error_message(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 3", + )) + .contains("unmatched_steps must be 0, 1, or 2") + ); + + // Rejected on an escalation route, including the implicit form where the + // `escalation` table alone selects the mode: the key is new, so no existing + // configuration relies on it being ignored there. + assert!( + error_message(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }", + )) + .contains("mode escalation cannot use capability routing settings") + ); + Ok(()) + } + #[test] fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> { let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\"";