diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 488bc6585..1b78dafd2 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -73,10 +73,15 @@ impl TaskClassifierVerdict { } /// Returns the number of threshold steps assigned to this capability boundary. - fn boundary_steps(&self) -> Option { + /// + /// `unmatched_steps` comes from the route configuration: a request that matches no + /// capability rule has no evidence behind its solve probability, so operators can + /// require more confidence for it than for an uncertain verdict. + fn boundary_steps(&self, unmatched_steps: u8) -> Option { match self.capability_boundary.as_str() { "supported" => Some(0), - "uncertain" | "unmatched" => Some(1), + "uncertain" => Some(1), + "unmatched" => Some(unmatched_steps), "unsupported" => Some(2), _ => None, } @@ -220,6 +225,7 @@ impl ClassifierInput for TaskInput { struct TaskClassifierPolicy { base_threshold: f64, threshold_step: f64, + unmatched_steps: u8, } impl TaskClassifierPolicy { @@ -227,12 +233,14 @@ impl TaskClassifierPolicy { Self { base_threshold: config.base_threshold, threshold_step: config.threshold_step, + unmatched_steps: config.unmatched_steps, } } /// Returns the required solve probability for one validated verdict. fn threshold(&self, verdict: &TaskClassifierVerdict) -> Option { - Some(self.base_threshold + f64::from(verdict.boundary_steps()?) * self.threshold_step) + let steps = verdict.boundary_steps(self.unmatched_steps)?; + Some(self.base_threshold + f64::from(steps) * self.threshold_step) } } @@ -304,9 +312,13 @@ pub struct TaskClassifierConfig { pub base_threshold: f64, /// Amount added per capability-boundary step. /// - /// Supported verdicts use `base_threshold`, uncertain and unmatched verdicts use one - /// step, and unsupported verdicts use two steps. + /// Supported verdicts use `base_threshold`, uncertain verdicts use one step, unmatched + /// verdicts use `unmatched_steps`, and unsupported verdicts use two steps. pub threshold_step: f64, + /// Threshold steps applied to an unmatched verdict (no capability rule applies). + /// + /// Defaults to 1, the same as an uncertain verdict. 2 treats it like unsupported. + pub unmatched_steps: u8, /// How often the classifier re-decides this session's target. pub classify_trigger: ClassifyTrigger, /// Uses the first user message as the SessionKey for sticky routing when session metadata is unavailable. @@ -331,6 +343,8 @@ struct TaskClassifierConfigWire { base_threshold: f64, #[serde(default)] threshold_step: f64, + #[serde(default = "default_unmatched_steps")] + unmatched_steps: u8, #[serde(default)] classify_trigger: ClassifyTrigger, #[serde(default)] @@ -359,6 +373,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { Ok(Self { base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, + unmatched_steps: wire.unmatched_steps, classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, @@ -368,6 +383,13 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { } } +/// Unmatched verdicts share the uncertain step unless the route says otherwise. +pub const DEFAULT_UNMATCHED_STEPS: u8 = 1; + +const fn default_unmatched_steps() -> u8 { + DEFAULT_UNMATCHED_STEPS +} + const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } @@ -377,6 +399,7 @@ impl Default for TaskClassifierConfig { Self { base_threshold: 0.0, threshold_step: 0.0, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, @@ -405,6 +428,14 @@ impl TaskClassifierConfig { ), }); } + if self.unmatched_steps > 2 { + return Err(LibsyError::AlgorithmError { + message: format!( + "unmatched_steps must be 0, 1, or 2, got {}", + self.unmatched_steps + ), + }); + } let unsupported_threshold = self.base_threshold + 2.0 * self.threshold_step; if unsupported_threshold > 1.0 && unsupported_threshold - 1.0 > f64::EPSILON { return Err(LibsyError::AlgorithmError { @@ -1378,6 +1409,49 @@ mod tests { Ok(()) } + #[test] + fn unmatched_steps_raise_only_the_unmatched_threshold() -> Result<()> { + let policy = TaskClassifierPolicy::new(&TaskClassifierConfig { + threshold_step: 0.1, + unmatched_steps: 2, + ..test_config(0.75) + }); + + // 0.85 clears one step but not two: unmatched now routes to the capable target. + assert_eq!( + selected(&policy, Some(&verdict(0.85, "unmatched", "none")))?, + "capable" + ); + assert_eq!( + selected(&policy, Some(&verdict(0.95, "unmatched", "none")))?, + "efficient" + ); + // Other boundaries keep their steps. + assert_eq!( + selected(&policy, Some(&verdict(0.85, "uncertain", "UNC-1")))?, + "efficient" + ); + assert_eq!( + selected(&policy, Some(&verdict(0.75, "supported", "SUP-2")))?, + "efficient" + ); + Ok(()) + } + + #[test] + fn unmatched_steps_default_to_one_and_reject_values_above_two() { + let parsed: TaskClassifierConfig = + serde_json::from_value(serde_json::json!({ "base_threshold": 0.5 })) + .expect("config without unmatched_steps parses"); + assert_eq!(parsed.unmatched_steps, 1); + + let invalid = TaskClassifierConfig { + unmatched_steps: 3, + ..test_config(0.5) + }; + assert!(invalid.validate().is_err()); + } + /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index a1f1ef022..0cabc4b14 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -20,8 +20,8 @@ mod algorithms; pub use algorithms::advisor_gate::{AdvisorGate, AdvisorGateConfig, GateTrigger}; pub use algorithms::composite::{CompositeRouter, CompositeRouterConfig}; pub use algorithms::llm_class::{ - CustomClassifierConfig, CustomClassifierPolicy, LlmClassifierConfig, LlmTaskClassifier, - TaskClassifierConfig, + CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, LlmClassifierConfig, + LlmTaskClassifier, TaskClassifierConfig, }; pub use algorithms::noop::Noop; pub use algorithms::passthrough::Passthrough; diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 23c90d7ad..31b003e10 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -14,10 +14,11 @@ use pyo3::prelude::*; use serde_json::Value; use switchyard_libsy::{ Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, - CustomClassifierConfig, CustomClassifierPolicy, DeescalationConfig, EscalationJudgeConfig, - HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, - LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, StageRouter, - StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, ToolSemantics, + CustomClassifierConfig, CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, DeescalationConfig, + EscalationJudgeConfig, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, + LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, RoutingOutcome, RuntimeModels, + StageRouter, StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, + ToolSemantics, }; use switchyard_protocol::{ Category, LlmClientError, LlmResponse, LlmResponseStream, LlmResponseStreamEvent, Metadata, @@ -330,6 +331,7 @@ impl PyTaskClassifierConfig { inner: TaskClassifierConfig { base_threshold, threshold_step, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index e742d48ba..f2d5c9276 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -12,10 +12,10 @@ use std::sync::Arc; use libsy::{ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, - CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, - PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, SubagentRouter, - SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, + CustomClassifierPolicy, DEFAULT_UNMATCHED_STEPS, EscalationJudgeConfig, GateTrigger, + HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, + PickerMode, PlanExecute, PlanExecuteConfig, Random, StageRouter, StageRouterConfig, + SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, ToolSemantics, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -103,6 +103,7 @@ struct CapabilityClassifierRouteConfig { weak_target: String, base_threshold: f64, threshold_step: f64, + unmatched_steps: u8, classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, @@ -237,6 +238,9 @@ pub struct LlmClassifierRouteConfig { /// Capability mode: how much to raise the threshold when the judge is /// uncertain. Added once for an uncertain verdict and twice for unsupported. pub threshold_step: Option, + /// Capability mode: threshold steps for an unmatched verdict (no capability rule + /// applies). 1 by default, the same as uncertain; 2 treats it like unsupported. + pub unmatched_steps: Option, /// How often the judge runs: every request, once per user turn, or once per session. pub classify_trigger: ClassifyTrigger, /// Reuses the session's target by hashing the first user message when no @@ -524,6 +528,7 @@ impl StageClassifierConfig { TaskClassifierConfig { base_threshold: self.base_threshold, threshold_step: self.threshold_step, + unmatched_steps: DEFAULT_UNMATCHED_STEPS, classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, @@ -878,6 +883,7 @@ impl LlmClassifierRouteConfig { weak_target, base_threshold, threshold_step, + unmatched_steps, classify_trigger, message_hash_fallback, recent_turn_window, @@ -933,6 +939,7 @@ impl LlmClassifierRouteConfig { base_threshold, )?, threshold_step: threshold_step.unwrap_or_default(), + unmatched_steps: unmatched_steps.unwrap_or(DEFAULT_UNMATCHED_STEPS), classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, @@ -956,11 +963,15 @@ impl LlmClassifierRouteConfig { "llm_classifier route {route_name} mode escalation cannot use classify_trigger" ))); } - if mode.is_some() - && (base_threshold.is_some() - || threshold_step.is_some() - || *message_hash_fallback - || recent_turn_window.is_some()) + // `unmatched_steps` is new, so no existing escalation configuration carries it: + // reject it even when the mode is implied by `escalation`. The older capability + // keys stay tolerated in that implicit form for compatibility. + if unmatched_steps.is_some() + || (mode.is_some() + && (base_threshold.is_some() + || threshold_step.is_some() + || *message_hash_fallback + || recent_turn_window.is_some())) { return Err(AlgorithmConfigError::new(format!( "llm_classifier route {route_name} mode escalation cannot use capability routing settings" @@ -992,6 +1003,7 @@ impl LlmClassifierRouteConfig { || weak_target.is_some() || base_threshold.is_some() || threshold_step.is_some() + || unmatched_steps.is_some() || escalation.is_some() || *response_format_type != ClassifierResponseFormat::JsonSchema { @@ -1231,6 +1243,7 @@ fn build_algorithm( let classifier_config = TaskClassifierConfig { base_threshold: config.base_threshold, threshold_step: config.threshold_step, + unmatched_steps: config.unmatched_steps, classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index f37b4cda9..e732dd90d 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1183,6 +1183,34 @@ new = ["send_message"] Ok(()) } + #[test] + fn unmatched_steps_is_a_capability_setting() -> RunnerResult<()> { + // Accepted on a capability route; the value range is checked by the algorithm. + runner_from_toml(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 2", + ))?; + assert!( + error_message(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 3", + )) + .contains("unmatched_steps must be 0, 1, or 2") + ); + + // Rejected on an escalation route, including the implicit form where the + // `escalation` table alone selects the mode: the key is new, so no existing + // configuration relies on it being ignored there. + assert!( + error_message(&VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nunmatched_steps = 2\nescalation = { confirmations = 2 }", + )) + .contains("mode escalation cannot use capability routing settings") + ); + Ok(()) + } + #[test] fn a_target_reasoning_effort_parses_and_is_rejected_where_unsupported() -> RunnerResult<()> { let strong = "[targets.strong]\nid = \"strong/model\"\nllm_client = \"responses\""; diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 152dbfe02..78b98cf4a 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -129,7 +129,8 @@ routes to `weak_target` or `strong_target`. Beyond the three targets it accepts | Key | Default | Meaning | |---|---|---| | `base_threshold` | *required* | Lowest solve probability that routes a task to `weak_target`. Raise it to send less traffic to the weak model. | -| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `threshold_step` | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `unmatched_steps` | `1` | Threshold steps for an unmatched verdict. `0`, `1`, or `2`; `2` treats it like unsupported. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request including tool continuations, `user_turn` judges each new user message and holds that target across the tool calls between, `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | Extends affinity to clients that send no session header, keying on the first user message. Requires `classify_trigger = "new_session"` or `"user_turn"`. | diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 8bf382a08..88c1ea0c8 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -239,7 +239,8 @@ Capability mode classifies before serving. See | `strong_target` | Yes | — | Capable tier. | | `weak_target` | Yes | — | Efficient tier. | | `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. | -| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain or unmatched verdicts and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. | +| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. | | `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | No | `false` | Retains the target against a hash of the first user message when a request carries no session ID. Requires `classify_trigger = "new_session"` or `"user_turn"`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index de35015ff..069a6139f 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -68,7 +68,8 @@ greater than or equal to the applicable threshold. Otherwise it routes to `strong_target`: - `supported` uses `base_threshold`. -- `uncertain` and `unmatched` use `base_threshold + threshold_step`. +- `uncertain` uses `base_threshold + threshold_step`. +- `unmatched` uses `base_threshold + unmatched_steps * threshold_step` (`unmatched_steps` defaults to `1`). - `unsupported` uses `base_threshold + 2 * threshold_step`. An invalid, inconsistent, or unparseable verdict routes to @@ -119,6 +120,7 @@ for the server merge behavior. |---|---|---| | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | +| `unmatched_steps` | `1` | Boundary steps applied to an `unmatched` verdict. `0`, `1`, or `2`; `2` requires the same confidence as an `unsupported` verdict. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"` or `"user_turn"`. |