From fd07bf33991ea6b49bd27630a17a3b8ac3929138 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Sun, 4 Oct 2026 20:28:21 -0700 Subject: [PATCH] feat(protocol): classify host client failures Signed-off-by: Alex Fournier --- crates/libsy-llm-client/src/observability.rs | 15 ++++++++- .../libsy-llm-client/tests/observability.rs | 33 +++++++++++++++++++ .../libsy/src/algorithms/util/robustness.rs | 8 +++-- crates/protocol/src/client.rs | 17 ++++++++-- crates/switchyard-runner/src/failure.rs | 4 ++- 5 files changed, 70 insertions(+), 7 deletions(-) diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs index 03a32951c..bfc5e8fb5 100644 --- a/crates/libsy-llm-client/src/observability.rs +++ b/crates/libsy-llm-client/src/observability.rs @@ -77,6 +77,10 @@ pub(crate) fn observe_client_call(result: Result) -> Result "client call to target {target:?} failed: upstream HTTP {status}" ), ), + LibsyError::ClientCall { + source: LlmClientError::Host { .. }, + .. + } => record_host_error(&span), _ => record_client_error(&span, &error_type, &error), } Err(error) @@ -206,6 +210,7 @@ fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> { LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"), LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.as_str().to_owned()), LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"), + LlmClientError::Host { .. } => Cow::Borrowed("host"), LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"), _ => Cow::Borrowed("_OTHER"), } @@ -279,7 +284,10 @@ impl ClientStreamObserver { } Err(error) => { let error_type = llm_client_error_type(error); - record_client_error(&self.span, &error_type, error); + match error { + LlmClientError::Host { .. } => record_host_error(&self.span), + _ => record_client_error(&self.span, &error_type, error), + } self.outcome = Outcome::Failed; } } @@ -326,6 +334,11 @@ impl Drop for ClientStreamObserver { } } +// Host sources are opaque and may contain caller-owned request data. +fn record_host_error(span: &Span) { + record_client_error(span, "host", &format_args!("host client error")); +} + fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) { span.record("outcome", "error"); span.record("otel.status_code", "ERROR"); diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 30366dbb2..de0c57792 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -1427,6 +1427,17 @@ impl RoutedLlmClient for TimeoutClient { } } +struct HostErrorClient; + +#[async_trait] +impl RoutedLlmClient for HostErrorClient { + async fn call(&self, _request: Request) -> Result { + Err(LlmClientError::Host { + source: Box::new(TestError(LEAKED_CONTENT)), + }) + } +} + #[tokio::test] async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Result<()> { let _guard = serialize_test().lock().await; @@ -1554,6 +1565,28 @@ async fn typed_client_failure_records_semantic_error_type() { ); } +/// Opaque host errors retain their source for the caller, not for telemetry. +#[tokio::test] +async fn host_error_source_is_redacted_from_the_client_call_span() { + let _guard = serialize_test().lock().await; + let (store, _, _, _, _) = telemetry(); + const MODEL: &str = "obs-host-error-model"; + let _ = run( + algo("obs-host-error-algo", MODEL), + Arc::new(HostErrorClient), + request_with_metadata("obs-host-error-session", "obs-host-error-corr"), + ) + .await; + + let spans = store.spans(); + let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL); + assert_eq!( + client_span.fields.get("error.type").map(String::as_str), + Some("host") + ); + assert!(!format!("{client_span:?}").contains(LEAKED_CONTENT)); +} + /// Verifies that the client span retains the HTTP status without the upstream body. #[tokio::test] async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_libsy::Result<()> { diff --git a/crates/libsy/src/algorithms/util/robustness.rs b/crates/libsy/src/algorithms/util/robustness.rs index ed51518a2..e06af3af1 100644 --- a/crates/libsy/src/algorithms/util/robustness.rs +++ b/crates/libsy/src/algorithms/util/robustness.rs @@ -48,8 +48,8 @@ pub(crate) fn safe_error_summary(error: &LibsyError) -> String { /// The client half of [`safe_error_summary`]. /// /// `UpstreamHttp` is the sharp edge: its `Display` interpolates the raw upstream -/// body, which routinely quotes the request back. Boxed transport, decode, and FFI -/// sources are reduced for the same reason. +/// body, which routinely quotes the request back. Boxed transport, decode, host, and +/// FFI sources are reduced for the same reason. pub(crate) fn safe_client_error(error: &LlmClientError) -> String { match error { LlmClientError::UpstreamHttp { status, .. } => format!("upstream HTTP {status}"), @@ -59,6 +59,7 @@ pub(crate) fn safe_client_error(error: &LlmClientError) -> String { LlmClientError::Timeout { .. } => "upstream request timed out".to_string(), LlmClientError::Transport { .. } => "upstream transport error".to_string(), LlmClientError::InvalidResponse { .. } => "invalid upstream response".to_string(), + LlmClientError::Host { .. } => "host client error".to_string(), LlmClientError::Ffi { .. } => "foreign function interface error".to_string(), LlmClientError::InvalidRequest { .. } => "invalid request".to_string(), LlmClientError::RequestTranslation(_) => "request translation failed".to_string(), @@ -115,6 +116,9 @@ mod tests { LlmClientError::Timeout { source: std::io::Error::other(SECRET).into(), }, + LlmClientError::Host { + source: std::io::Error::other(SECRET).into(), + }, ] { let summary = safe_client_error(&source); assert!(!summary.contains(SECRET), "{summary}"); diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs index fa37f01c1..15724b890 100644 --- a/crates/protocol/src/client.rs +++ b/crates/protocol/src/client.rs @@ -17,8 +17,8 @@ pub type BoxError = Box; /// Failures a routed LLM client can surface to its caller. /// /// The variants classify failures that routing hosts commonly need to handle, -/// while boxed sources preserve implementation-specific detail. `General` is the -/// escape hatch for failures that do not fit a shared category. +/// while boxed sources preserve implementation-specific detail. `Host` preserves +/// an opaque native host error; `General` is the string-only escape hatch. #[non_exhaustive] #[derive(Debug, Error)] pub enum LlmClientError { @@ -107,6 +107,17 @@ pub enum LlmClientError { source: BoxError, }, + /// An opaque embedding-host failure that does not fit a shared category. + /// + /// The default client runner treats this as terminal. Use a transport, timeout, HTTP, + /// or context-window variant when one applies. + #[error("host client error: {source}")] + Host { + /// Original host error, preserved for same-process recovery. + #[source] + source: BoxError, + }, + /// A call across a foreign-function boundary (e.g. a Python-implemented client) /// failed. The boxed source is the foreign error itself. #[error("foreign function interface error: {source}")] @@ -116,7 +127,7 @@ pub enum LlmClientError { source: BoxError, }, - /// A string message. Useful in testing, but prefer adding variants over using this. + /// A string-only failure. Useful in testing; prefer a typed variant when possible. #[error("{0}")] General(String), } diff --git a/crates/switchyard-runner/src/failure.rs b/crates/switchyard-runner/src/failure.rs index e73b05a29..9d11f0d66 100644 --- a/crates/switchyard-runner/src/failure.rs +++ b/crates/switchyard-runner/src/failure.rs @@ -155,7 +155,9 @@ fn client_error_summary( LlmClientError::ResponseTranslation(_) => (RouteErrorKind::ResponseTranslation, None), LlmClientError::InvalidRequest { .. } => (RouteErrorKind::InvalidRequest, None), LlmClientError::Configuration { .. } => (RouteErrorKind::Configuration, None), - LlmClientError::Ffi { .. } | LlmClientError::General(_) => (RouteErrorKind::Other, None), + LlmClientError::Host { .. } | LlmClientError::Ffi { .. } | LlmClientError::General(_) => { + (RouteErrorKind::Other, None) + } _ => (RouteErrorKind::Other, None), }; summary(kind, phase, upstream_status, target)