Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 29 additions & 21 deletions crates/switchyard-runner/src/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,21 +105,26 @@ impl RunnerError {
}
}

/// A configured algorithm and the per-target clients its calls resolve through.
pub struct Route {
// State that can select or call a model during one route execution.
struct ExecutionLane {
algorithm: Arc<dyn Algorithm>,
// Resolves each offloaded call to the client configured for the target the algorithm
// selected. A route is a synthetic model with no upstream of its own, so this is a
// per-target lookup, never one client serving the whole route.
clients: ClientRouter,
caller_auth: Option<CallerAuthKind>,
capabilities: ModelCapabilities,
anthropic_auxiliary_target: Option<AuxiliaryTarget>,
responses_auxiliary_target: Option<AuxiliaryTarget>,
decision_targets: Vec<DecisionTarget>,
models: Arc<RuntimeModels>,
}

/// A configured algorithm and the per-target clients its calls resolve through.
pub struct Route {
lane: ExecutionLane,
caller_auth: Option<CallerAuthKind>,
capabilities: ModelCapabilities,
}

/// The selected model and untouched response produced by a route execution.
pub struct RunOutput {
pub selected_model: ModelId,
Expand All @@ -140,20 +145,22 @@ impl Route {
models: RuntimeModels,
) -> Self {
Self {
algorithm,
clients,
lane: ExecutionLane {
algorithm,
clients,
anthropic_auxiliary_target,
responses_auxiliary_target,
decision_targets,
models: Arc::new(models),
},
caller_auth,
capabilities,
anthropic_auxiliary_target,
responses_auxiliary_target,
decision_targets,
models: Arc::new(models),
}
}

/// Returns the configured libsy algorithm name.
pub fn algorithm_name(&self) -> &str {
self.algorithm.name()
self.lane.algorithm.name()
}

/// Returns model-list capability metadata.
Expand All @@ -168,15 +175,16 @@ impl Route {

/// Resolves a selected model to this route's non-secret target metadata.
pub(crate) fn decision_target(&self, model: &ModelId) -> Option<DecisionTarget> {
self.decision_targets
self.lane
.decision_targets
.iter()
.find(|target| target.model == *model)
.cloned()
}

/// Returns the models grouped for one algorithm execution.
pub fn models(&self) -> &RuntimeModels {
&self.models
&self.lane.models
}

/// Rejects a caller format incompatible with forwarded credentials.
Expand All @@ -196,10 +204,10 @@ impl Route {
observer: Option<RunObserver>,
) -> Result<RunOutput, RunnerError> {
let (selected_model, response) = switchyard_llm_client::run(
Arc::clone(&self.algorithm),
self.clients.clone(),
Arc::clone(&self.lane.algorithm),
self.lane.clients.clone(),
request,
Arc::clone(&self.models),
Arc::clone(&self.lane.models),
observer,
)
.await?;
Expand All @@ -212,10 +220,10 @@ impl Route {
/// Completes routing-time calls without serving a post-routing completion.
pub async fn decide(&self, request: Request) -> Result<RoutingOutcome, RunnerError> {
switchyard_llm_client::decide(
Arc::clone(&self.algorithm),
self.clients.clone(),
Arc::clone(&self.lane.algorithm),
self.lane.clients.clone(),
request,
Arc::clone(&self.models),
Arc::clone(&self.lane.models),
)
.await
.map_err(Into::into)
Expand All @@ -228,9 +236,9 @@ impl Route {
operation: AuxiliaryOperation,
) -> Result<Value, RunnerError> {
let target = match operation {
AuxiliaryOperation::AnthropicCountTokens => &self.anthropic_auxiliary_target,
AuxiliaryOperation::AnthropicCountTokens => &self.lane.anthropic_auxiliary_target,
AuxiliaryOperation::ResponsesInputTokens | AuxiliaryOperation::ResponsesCompact => {
&self.responses_auxiliary_target
&self.lane.responses_auxiliary_target
}
}
.as_ref()
Expand Down
Loading