From bbdccf963e5e0ec655c7675a99dfaf4fa588c7fd Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sun, 2 Aug 2026 22:16:38 +0100 Subject: [PATCH 1/2] feat(modalities): add bounded capability contracts [roadmap:WF-ROADMAP-0010] --- CHANGELOG.md | 6 + README.md | 2 + ...F-ADR-0056-hard-destination-eligibility.md | 8 +- .../WF-ADR-0067-bounded-modality-contracts.md | 50 +++++++ docs/gateway-config.md | 10 +- docs/integrations.md | 10 +- docs/rust-migration-capability-matrix.md | 1 + ...010-verified-efficiency-evidence-engine.md | 8 ++ rust/crates/wayfinder-apple-ffi/src/lib.rs | 50 +++++++ rust/crates/wayfinder-gateway/src/lib.rs | 67 +++++++++- .../wayfinder-gateway/src/modalities.rs | 126 ++++++++++++++++++ rust/crates/wayfinder-routing-core/src/lib.rs | 65 ++++++++- .../wayfinder-runtime-contracts/src/lib.rs | 51 +++++++ 13 files changed, 435 insertions(+), 19 deletions(-) create mode 100644 decisions/WF-ADR-0067-bounded-modality-contracts.md create mode 100644 rust/crates/wayfinder-gateway/src/modalities.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index dec0461..06aec87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ details, release history over commit history. ### Added +- **Bounded modality capability contracts** (WF-ADR-0067, issue #154). The shared Rust routing + contract now carries explicit embeddings, image-generation, audio, and batch capabilities and + applies them as hard pre-score eligibility filters. The current gateway remains text-only and + rejects known modality payloads before provider delivery; each non-text surface will be enabled + independently only after its adapter, bounds, accounting, and parity fixtures pass. + - **Bounded OpenAI Responses API compatibility** (WF-ADR-0066, issue #153). The Rust gateway now exposes authenticated `/v1/responses` and `/responses` aliases with strict text/multi-turn input, normalized usage, bounded SSE streaming, one terminal event, and explicit unsupported-field diff --git a/README.md b/README.md index 5718b5f..286562a 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ The gateway exposes: - OpenAI-compatible: `http://127.0.0.1:8088/v1` - OpenAI Responses compatibility: `POST /v1/responses` (bounded text and multi-turn contract) +- Modality compatibility: explicit embeddings, image, audio, and batch capability contracts; + non-text surfaces remain fail-closed until their reviewed adapters are enabled - Anthropic-compatible: `http://127.0.0.1:8088` - Health: `http://127.0.0.1:8088/healthz` diff --git a/decisions/WF-ADR-0056-hard-destination-eligibility.md b/decisions/WF-ADR-0056-hard-destination-eligibility.md index c5cc754..0763dad 100644 --- a/decisions/WF-ADR-0056-hard-destination-eligibility.md +++ b/decisions/WF-ADR-0056-hard-destination-eligibility.md @@ -50,8 +50,9 @@ control contract when entering the shared chat path. Request requirements are derived from the bounded OpenAI-compatible body: estimated prompt plus requested output context, image content, tool/function declarations, and streaming. Provider defaults are conservative for the two -text-only native adapters; OpenAI-compatible delivery retains its existing -pass-through multimodal/tool contract. +text-only native adapters; non-text surfaces are additionally governed by the +explicit capability contract in WF-ADR-0067 and fail closed until a reviewed +adapter is enabled. Tool declarations remain an explicit provider capability. Explicit pins and named `@route/` presets fail closed with `422 wayfinder_router_destination_ineligible` when no eligible destination @@ -69,7 +70,8 @@ remain unchanged. - A provider with unknown native readiness is not guessed ready; missing credentials are excluded before delivery. - Generic OpenAI-compatible endpoints remain backward-compatible for their - existing text, image, tool, and streaming pass-through requests. + existing text, tool, and streaming requests; non-text payloads are rejected + until their modality adapters are independently enabled. - Explicit pins may now return a specific eligibility error instead of a provider-side malformed-request or privacy leak. diff --git a/decisions/WF-ADR-0067-bounded-modality-contracts.md b/decisions/WF-ADR-0067-bounded-modality-contracts.md new file mode 100644 index 0000000..823dfb7 --- /dev/null +++ b/decisions/WF-ADR-0067-bounded-modality-contracts.md @@ -0,0 +1,50 @@ +# WF-ADR-0067: Bounded modality capability contracts + +- Status: accepted for incremental implementation +- Date: 2026-08-02 +- Roadmap: `WF-ROADMAP-0010` +- Issue: #154 + +## Decision + +Represent non-text execution as an explicit `InferenceSurface` in the shared +runtime contract. Destination snapshots carry independent, secret-free +capabilities for embeddings, image generation, audio input/output, and batch +execution. The routing core applies those capabilities as hard exclusions +before deterministic scoring; a score cannot make an incompatible destination +eligible. + +The current Rust gateway advertises and executes text surfaces only. The new +capability flags default to `false`, and the existing provider adapters do not +opt into them. A known embeddings, image, audio, or batch-shaped payload sent +to the text endpoint fails with `wayfinder_router_unsupported_modality` before +provider delivery. No surface is advertised merely because a provider happens +to accept an OpenAI-shaped request. + +## Incremental enablement + +Each surface gets its own provider adapter and parity fixtures before being +enabled: + +1. embeddings — bounded input arrays, response vectors, dimensions, usage, and + provider-specific cost accounting; +2. image — strict binary/input and generated-output bounds; +3. audio — input/output format, duration, streaming, and retention bounds; +4. batch — durable state, cancellation, expiry, reconciliation, and batch + accounting. + +The surfaces remain independently shippable. Image, audio, and batch are not +implicitly enabled by the embeddings work, and no batch persistence exists in +this slice. + +## Consequences + +- Swift/FFI hosts and the gateway share one capability vocabulary and stable + exclusion reason names. +- Unsupported modality content cannot be silently forwarded to a text-only + provider or counted as a successful text turn. +- Existing text routing remains wire-compatible; the default contract is + unchanged for ordinary Chat Completions and Responses requests. +- Provider adapters must explicitly opt in only after request/response bounds, + concurrency/deadline policy, usage normalization, cost classification, and + gateway-versus-adapter parity tests are complete. diff --git a/docs/gateway-config.md b/docs/gateway-config.md index 2c2f7b3..d81ba9f 100644 --- a/docs/gateway-config.md +++ b/docs/gateway-config.md @@ -269,8 +269,8 @@ may select a privacy boundary with `x-wayfinder-privacy-posture`: The body also contributes hard requirements: estimated prompt plus requested output context, image content, tool/function declarations, and streaming. Apple Foundation Models and the bounded ChatGPT adapter are text-only and are -excluded for image/tool requests; OpenAI-compatible adapters retain their -existing pass-through contract. Missing credentials, declared windows that are +excluded for image/tool requests; the gateway's explicit modality gate also +rejects non-text payloads before any OpenAI-compatible delivery. Missing credentials, declared windows that are too small, unsupported capabilities, and denied privacy boundaries are excluded before reliability retries or failover. Models that omit `context_window` retain the legacy prompt-precheck behavior. Every concrete @@ -280,6 +280,12 @@ preset with no eligible member returns `422 wayfinder_router_destination_ineligible` with stable reason names rather than silently switching privacy boundary (WF-ADR-0056). +Non-text execution surfaces are separately capability-gated. The shared +contract names embeddings, image generation, audio, and batch support, but the +current gateway defaults every one to unsupported. Known non-text payloads fail +before provider delivery rather than being forwarded through a text adapter; +see WF-ADR-0067 for the independent enablement sequence. + OpenAI- and Anthropic-compatible chat endpoints preserve the same bounded Wayfinder control-header allowlist. In particular, privacy and offline controls cannot disappear when an Anthropic-shaped request enters the shared router. diff --git a/docs/integrations.md b/docs/integrations.md index 18c0f47..1bb7e6c 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -168,8 +168,14 @@ thinking are not translated yet (WF-DESIGN-0011). `model`, `input`, `instructions`, `stream`, `max_output_tokens`, and `temperature` contract. Unsupported fields fail specifically instead of being dropped; see [WF-ADR-0066](../decisions/WF-ADR-0066-bounded-responses-api.md). -- **Tool calling / vision** depend on the *upstream* model you route to, not on Wayfinder — - the gateway forwards your request body unchanged (plus the resolved model id). +- **Non-text surfaces** are currently fail-closed. Embeddings, image, audio, and batch payloads + are not advertised or forwarded until their provider adapter, request/response bounds, usage + accounting, and parity fixtures are enabled; known modality fields sent to the text endpoint + return `wayfinder_router_unsupported_modality` before provider delivery. See + [WF-ADR-0067](../decisions/WF-ADR-0067-bounded-modality-contracts.md). +- **Tool calling** depends on the *upstream* model you route to, while vision/audio/image + surfaces remain disabled until their reviewed adapters are enabled; known non-text payloads + are rejected by the modality gate rather than forwarded unchanged. - **Per-request overrides** travel as headers (e.g. `X-Wayfinder-Threshold`), so you can tune routing without changing client config. diff --git a/docs/rust-migration-capability-matrix.md b/docs/rust-migration-capability-matrix.md index 88af890..7d3f38c 100644 --- a/docs/rust-migration-capability-matrix.md +++ b/docs/rust-migration-capability-matrix.md @@ -147,6 +147,7 @@ verified subset; the proposed owner is an architectural mapping, not permission | Config preview and feedback endpoints | `POST /router/config`, `/v1/feedback` | gateway/UI/feedback tests | `routes::{config_preview,feedback}` | Config preview is constrained/read-only; feedback may require an environment token but is open by default and persists raw prompt labels. Critical auth/storage risk. | | OpenAI chat aliases | `POST /v1/chat/completions`, `/chat/completions` | `test_gateway.py` | `routes::chat` | Real upstream success is relayed, routing headers added, and upstream headers mostly discarded. Bare alias is a compatibility contract. Critical wire parity. | | OpenAI Responses aliases | `POST /v1/responses`, `/responses` | gateway Responses contract tests, WF-ADR-0066 | `wayfinder-gateway::responses` | Strict bounded text/multi-turn adapter over the authenticated Chat Completions path; normalized usage and SSE events; unsupported fields fail specifically. Critical compatibility/bounds contract. | +| Modality capability contracts | embeddings, image, audio, batch capability flags and hard exclusions | routing-core/FFI/gateway modality tests, WF-ADR-0067 | `wayfinder-runtime-contracts`, `wayfinder-routing-core`, `wayfinder-apple-ffi`, `wayfinder-gateway::modalities` | Non-text surfaces are independently gated and currently fail-closed before provider delivery; no surface is advertised without an adapter and parity fixtures. High compatibility/bounds risk. | | Request IDs and routing headers | chat handler | gateway tests | `wayfinder-gateway::response_meta` | 12 lower-case hex ID; score is two decimals; model/mode and optional served-by/failover/budget/offline/cache/decision-only plus rate headers. Critical exact-header parity. | | Route scope | `extract_prompt` | gateway tests, ADR-0021 | `decision_policy::scope` | `turn`, `last_user`, `user`, `all`; text parts joined; role-filter fallback to last message. High multi-turn parity. | | Pin, threshold, sticky, tuning, slash overrides | gateway policy helpers | gateway tests, ADRs 0011/0022/0023/0036 | `decision_policy` | Pin > slash pin > threshold/scoring/sticky, with exact accepted aliases and modes. Unknown model IDs score normally. Critical precedence parity. | diff --git a/roadmaps/WF-ROADMAP-0010-verified-efficiency-evidence-engine.md b/roadmaps/WF-ROADMAP-0010-verified-efficiency-evidence-engine.md index 9ca9aee..688f3d2 100644 --- a/roadmaps/WF-ROADMAP-0010-verified-efficiency-evidence-engine.md +++ b/roadmaps/WF-ROADMAP-0010-verified-efficiency-evidence-engine.md @@ -75,6 +75,14 @@ usage and streaming events, and fail closed on unsupported fields. Input, output, event, response, and accounting bounds are explicit; duplicate terminal events or a second route decision are not permitted. See WF-ADR-0066. +Issue #154 begins the modality compatibility sequence. The shared runtime and +Apple bridge now carry explicit embeddings, image-generation, audio, and batch +capabilities, and the gateway rejects known non-text payloads before delivery +while every flag remains disabled by default. Each surface will ship through a +separate provider adapter and parity/bounds/accounting gate; batch persistence +and reconciliation are intentionally not present in this foundation slice. +See WF-ADR-0067. + WF-ADR-0052 adds bounded workspace policy above virtual keys: shared process-local RPM/TPM, inherited model allowlists, stable public model aliases, and explicit workspace response attribution. It also fixes the enterprise diff --git a/rust/crates/wayfinder-apple-ffi/src/lib.rs b/rust/crates/wayfinder-apple-ffi/src/lib.rs index 37e7783..d7e3436 100644 --- a/rust/crates/wayfinder-apple-ffi/src/lib.rs +++ b/rust/crates/wayfinder-apple-ffi/src/lib.rs @@ -118,6 +118,11 @@ pub struct DestinationCapabilities { pub streaming: bool, pub image_input: bool, pub tools: bool, + pub embeddings: bool, + pub image_generation: bool, + pub audio_input: bool, + pub audio_output: bool, + pub batch: bool, } /// Requirements that must be satisfied before complexity scoring. @@ -127,6 +132,17 @@ pub struct RoutingRequirements { pub image_input: bool, pub tools: bool, pub streaming: bool, + pub surface: InferenceSurface, +} + +/// Provider execution surface required by a mobile request. +#[derive(Clone, Copy, Debug, uniffi::Enum)] +pub enum InferenceSurface { + Text, + Embeddings, + ImageGeneration, + Audio, + Batch, } /// Bounded request supplied by an Apple host. @@ -166,6 +182,10 @@ pub enum ExclusionReason { ImageInputUnsupported, ToolsUnsupported, StreamingUnsupported, + EmbeddingsUnsupported, + ImageGenerationUnsupported, + AudioUnsupported, + BatchUnsupported, AutomaticNotAllowed, } @@ -461,6 +481,19 @@ impl From for CoreRoutingRequirements { image_input: value.image_input, tools: value.tools, streaming: value.streaming, + surface: value.surface.into(), + } + } +} + +impl From for wayfinder_routing_core::InferenceSurface { + fn from(value: InferenceSurface) -> Self { + match value { + InferenceSurface::Text => Self::Text, + InferenceSurface::Embeddings => Self::Embeddings, + InferenceSurface::ImageGeneration => Self::ImageGeneration, + InferenceSurface::Audio => Self::Audio, + InferenceSurface::Batch => Self::Batch, } } } @@ -484,6 +517,13 @@ impl From for CoreDestinationCapabilities { streaming: value.streaming, image_input: value.image_input, tools: value.tools, + modalities: wayfinder_routing_core::ModalityCapabilities { + embeddings: value.embeddings, + image_generation: value.image_generation, + audio_input: value.audio_input, + audio_output: value.audio_output, + batch: value.batch, + }, } } } @@ -556,6 +596,10 @@ impl From for ExclusionReason { CoreExclusionReason::ImageInputUnsupported => Self::ImageInputUnsupported, CoreExclusionReason::ToolsUnsupported => Self::ToolsUnsupported, CoreExclusionReason::StreamingUnsupported => Self::StreamingUnsupported, + CoreExclusionReason::EmbeddingsUnsupported => Self::EmbeddingsUnsupported, + CoreExclusionReason::ImageGenerationUnsupported => Self::ImageGenerationUnsupported, + CoreExclusionReason::AudioUnsupported => Self::AudioUnsupported, + CoreExclusionReason::BatchUnsupported => Self::BatchUnsupported, CoreExclusionReason::AutomaticNotAllowed => Self::AutomaticNotAllowed, } } @@ -622,6 +666,11 @@ mod tests { streaming: true, image_input: false, tools: false, + embeddings: false, + image_generation: false, + audio_input: false, + audio_output: false, + batch: false, }, automatic_eligible: true, } @@ -638,6 +687,7 @@ mod tests { image_input: false, tools: false, streaming: true, + surface: InferenceSurface::Text, }, } } diff --git a/rust/crates/wayfinder-gateway/src/lib.rs b/rust/crates/wayfinder-gateway/src/lib.rs index 9fb6e39..5fe6fd2 100644 --- a/rust/crates/wayfinder-gateway/src/lib.rs +++ b/rust/crates/wayfinder-gateway/src/lib.rs @@ -19,6 +19,7 @@ pub mod delivery; pub mod deployment_selection; pub mod evidence; pub mod metrics; +pub mod modalities; pub mod operator_auth; pub mod otel; pub mod rate_limit; @@ -71,9 +72,10 @@ use wayfinder_providers::sse::{SseDecoder, SseEvent}; use wayfinder_routing_core::profiles::{LexiconProfile, profiles}; use wayfinder_routing_core::{ CandidateAssessment, ComplexityScore, DestinationCapabilities, DestinationSnapshot, - ExclusionReason, ExecutionBoundary, FeatureContribution, Features, PrivacyPosture, - ProviderReadiness, RoutingConfig, RoutingRequest, RoutingRequirements, Tier, - assess_destination, explain_score, python_round, recommend_tier, score_complexity, + ExclusionReason, ExecutionBoundary, FeatureContribution, Features, InferenceSurface, + ModalityCapabilities, PrivacyPosture, ProviderReadiness, RoutingConfig, RoutingRequest, + RoutingRequirements, Tier, assess_destination, explain_score, python_round, recommend_tier, + score_complexity, }; use wayfinder_service::pricing::{ LedgerError, PriceTable, SavingsLedger, SavingsReport, UtcDate, estimate_tokens, price_table, @@ -2577,6 +2579,14 @@ pub(crate) async fn chat_completions( Err(response) => return *response, }; let request_id = new_request_id(); + if let Some(message) = modalities::unsupported_text_surface(&body) { + return error_response( + StatusCode::BAD_REQUEST, + "wayfinder_router_unsupported_modality", + message, + request_id_headers(&request_id), + ); + } let access_grant = match preflight_access(&state, &headers, &request_id).await { Ok(grant) => grant, Err(response) => return *response, @@ -4993,14 +5003,15 @@ fn model_execution_boundary(model: &ConfiguredModel) -> ExecutionBoundary { fn model_capabilities(model: &ConfiguredModel) -> DestinationCapabilities { match model.provider() { - // The OpenAI-compatible adapter is deliberately a pass-through. Its - // request contract supports multimodal messages and tool declarations; - // provider-specific rejection still remains a delivery failure. + // The OpenAI-compatible adapter currently supports text, tools, and + // streaming. Non-text surfaces remain explicit opt-ins and are not + // advertised until their adapters and parity fixtures ship. ProviderKind::OpenAiCompatible => DestinationCapabilities { text: true, streaming: true, - image_input: true, + image_input: false, tools: true, + modalities: ModalityCapabilities::default(), }, // The bounded native adapters currently accept text-only turns. They // both expose ordered streaming, but reject images and tools before @@ -5011,6 +5022,7 @@ fn model_capabilities(model: &ConfiguredModel) -> DestinationCapabilities { streaming: true, image_input: false, tools: false, + modalities: ModalityCapabilities::default(), } } } @@ -5153,6 +5165,7 @@ fn request_requirements( image_input: request_requires_image_input(messages), tools: request_requires_tools(body), streaming: body.get("stream").and_then(Value::as_bool) == Some(true), + surface: InferenceSurface::Text, } } @@ -5196,6 +5209,10 @@ fn exclusion_reason_name(reason: ExclusionReason) -> &'static str { ExclusionReason::ImageInputUnsupported => "image-input-unsupported", ExclusionReason::ToolsUnsupported => "tools-unsupported", ExclusionReason::StreamingUnsupported => "streaming-unsupported", + ExclusionReason::EmbeddingsUnsupported => "embeddings-unsupported", + ExclusionReason::ImageGenerationUnsupported => "image-generation-unsupported", + ExclusionReason::AudioUnsupported => "audio-unsupported", + ExclusionReason::BatchUnsupported => "batch-unsupported", ExclusionReason::AutomaticNotAllowed => "automatic-not-allowed", } } @@ -8151,6 +8168,42 @@ mod tests { Ok(()) } + #[tokio::test] + async fn unsupported_modalities_fail_before_provider_delivery() -> TestResult { + let calls = Arc::new(AtomicUsize::new(0)); + let state = AppState::new( + RoutingConfig::binary(0.5), + vec![ConfiguredModel::new( + "local", + "http://127.0.0.1:11434/v1", + "provider-local", + None, + true, + )], + false, + "test", + ) + .with_delivery(Arc::new(CountingUsageDelivery { + calls: Arc::clone(&calls), + })); + + for payload in [ + json!({"input": "embed this"}), + json!({"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:"}}]}]}), + json!({"messages": [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "..."}}]}]}), + json!({"batch": {"id": "batch-1"}}), + ] { + let response = post_json(&state, "/v1/chat/completions", &payload, &[]).await?; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!( + json_body(response).await?["error"]["type"], + "wayfinder_router_unsupported_modality" + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 0); + Ok(()) + } + #[tokio::test] async fn pins_threshold_route_scope_and_offline_override_are_applied() -> TestResult { let state = AppState::new( diff --git a/rust/crates/wayfinder-gateway/src/modalities.rs b/rust/crates/wayfinder-gateway/src/modalities.rs new file mode 100644 index 0000000..51fdc2d --- /dev/null +++ b/rust/crates/wayfinder-gateway/src/modalities.rs @@ -0,0 +1,126 @@ +//! Fail-closed modality boundary for the gateway. +//! +//! The current gateway ships text execution only. This module makes that +//! boundary explicit so an embeddings, image, audio, or batch-shaped request +//! cannot be silently forwarded to a text provider while those adapters are +//! still being qualified. Each surface will be enabled independently once its +//! provider contract, bounds, accounting, and parity fixtures land. + +use serde_json::{Map, Value}; + +/// Version of the normalized modality contract. +pub const MODALITY_CONTRACT_VERSION: &str = "wf-modalities-v1"; + +const MAX_SCAN_DEPTH: usize = 64; + +/// Validate a Chat Completions body against the currently shipped text-only +/// surface. `None` means the body contains no known unsupported modality. +pub(crate) fn unsupported_text_surface(body: &Map) -> Option { + for field in [ + "input", + "images", + "image", + "image_url", + "audio", + "input_audio", + "audio_url", + "modalities", + "audio_config", + "voice", + "format", + "batch", + "batch_id", + "file_id", + "input_file", + "files", + ] { + if body.contains_key(field) { + return Some(format!( + "unsupported modality field '{field}'; this gateway build exposes text execution only" + )); + } + } + body.get("messages") + .and_then(|messages| find_nested_modality(messages, 0)) +} + +fn find_nested_modality(value: &Value, depth: usize) -> Option { + if depth >= MAX_SCAN_DEPTH { + return Some(format!( + "request nesting exceeds the modality validation depth of {MAX_SCAN_DEPTH}" + )); + } + match value { + Value::Array(values) => values + .iter() + .find_map(|value| find_nested_modality(value, depth.saturating_add(1))), + Value::Object(object) => { + for field in [ + "image_url", + "input_image", + "image", + "input_audio", + "audio", + "audio_url", + "input_file", + "file", + "file_id", + ] { + if object.contains_key(field) { + return Some(format!( + "unsupported modality field '{field}'; this gateway build exposes text execution only" + )); + } + } + object + .values() + .find_map(|value| find_nested_modality(value, depth.saturating_add(1))) + } + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => None, + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn object(value: Value) -> Map { + value.as_object().cloned().unwrap_or_default() + } + + #[test] + fn known_surfaces_fail_closed_before_delivery() { + for value in [ + json!({"input": "hello"}), + json!({"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:"}}]}]}), + json!({"messages": [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "..."}}]}]}), + json!({"batch": {"id": "batch-1"}}), + ] { + assert!(unsupported_text_surface(&object(value)).is_some()); + } + } + + #[test] + fn ordinary_text_body_is_accepted() { + let body = object(json!({ + "messages": [{"role": "user", "content": "hello"}], + "stream": false + })); + assert_eq!(unsupported_text_surface(&body), None); + } + + #[test] + fn nested_validation_has_a_depth_bound() { + let mut value = json!("text"); + for _ in 0..=MAX_SCAN_DEPTH { + value = json!([value]); + } + let body = object(json!({"messages": value})); + assert!( + unsupported_text_surface(&body) + .is_some_and(|message| message.contains("modality validation depth")) + ); + } +} diff --git a/rust/crates/wayfinder-routing-core/src/lib.rs b/rust/crates/wayfinder-routing-core/src/lib.rs index 3251014..4b694c9 100644 --- a/rust/crates/wayfinder-routing-core/src/lib.rs +++ b/rust/crates/wayfinder-routing-core/src/lib.rs @@ -16,9 +16,9 @@ use thiserror::Error; pub use wayfinder_runtime_contracts::{ BillingClass, CandidateAssessment, DestinationCapabilities, DestinationSnapshot, - ExclusionReason, ExecutionBoundary, PrivacyPosture, ProviderReadiness, - RUNTIME_CONTRACT_VERSION, RouteExplanation, RoutePlan, RouteReceipt, RoutingRequest, - RoutingRequirements, + ExclusionReason, ExecutionBoundary, InferenceSurface, ModalityCapabilities, PrivacyPosture, + ProviderReadiness, RUNTIME_CONTRACT_VERSION, RouteExplanation, RoutePlan, RouteReceipt, + RoutingRequest, RoutingRequirements, }; /// Default cut for the binary local/cloud router. @@ -786,8 +786,34 @@ pub fn assess_destination( { exclusions.push(ExclusionReason::PrivacyBoundaryDenied); } - if !destination.capabilities.text { - exclusions.push(ExclusionReason::TextUnsupported); + match request.requirements.surface { + InferenceSurface::Text => { + if !destination.capabilities.text { + exclusions.push(ExclusionReason::TextUnsupported); + } + } + InferenceSurface::Embeddings => { + if !destination.capabilities.modalities.embeddings { + exclusions.push(ExclusionReason::EmbeddingsUnsupported); + } + } + InferenceSurface::ImageGeneration => { + if !destination.capabilities.modalities.image_generation { + exclusions.push(ExclusionReason::ImageGenerationUnsupported); + } + } + InferenceSurface::Audio => { + if !destination.capabilities.modalities.audio_input + && !destination.capabilities.modalities.audio_output + { + exclusions.push(ExclusionReason::AudioUnsupported); + } + } + InferenceSurface::Batch => { + if !destination.capabilities.modalities.batch { + exclusions.push(ExclusionReason::BatchUnsupported); + } + } } if let Some(required) = request.requirements.context_tokens { match destination.context_window { @@ -1065,6 +1091,7 @@ mod tests { streaming: true, image_input: false, tools: false, + modalities: ModalityCapabilities::default(), }, automatic_eligible: true, } @@ -1083,6 +1110,34 @@ mod tests { } } + #[test] + fn modality_capabilities_are_hard_filters_before_scoring() { + let mut request = request(PrivacyPosture::HostedAllowed); + request.requirements.surface = InferenceSurface::Embeddings; + let text_only = destination("text-only", "local", ExecutionBoundary::Hosted); + let assessment = assess_destination(&request, &text_only); + assert_eq!( + assessment.exclusions, + vec![ExclusionReason::EmbeddingsUnsupported] + ); + + let mut embeddings = destination("embeddings", "local", ExecutionBoundary::Hosted); + embeddings.capabilities.modalities.embeddings = true; + assert!(assess_destination(&request, &embeddings).is_eligible()); + } + + #[test] + fn non_text_surface_contracts_round_trip_with_defaults() -> Result<(), CoreError> { + let mut request = request(PrivacyPosture::HostedAllowed); + request.requirements.surface = InferenceSurface::Batch; + let encoded = serde_json::to_string(&request) + .map_err(|_| CoreError::InvalidContract("encode".to_owned()))?; + let decoded: RoutingRequest = serde_json::from_str(&encoded) + .map_err(|_| CoreError::InvalidContract("decode".to_owned()))?; + assert_eq!(decoded.requirements.surface, InferenceSurface::Batch); + Ok(()) + } + #[test] fn python_round_matches_binary_half_even_traps() { assert_eq!(python_round(0.005, 2), 0.01); diff --git a/rust/crates/wayfinder-runtime-contracts/src/lib.rs b/rust/crates/wayfinder-runtime-contracts/src/lib.rs index 5741db2..adc0204 100644 --- a/rust/crates/wayfinder-runtime-contracts/src/lib.rs +++ b/rust/crates/wayfinder-runtime-contracts/src/lib.rs @@ -94,6 +94,43 @@ pub enum BillingClass { Unknown, } +/// A normalized execution surface. Surface selection is a provider +/// capability, not part of the deterministic complexity score. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum InferenceSurface { + /// Text generation (Chat Completions, Messages, or bounded Responses). + #[default] + Text, + /// Vector embeddings. + Embeddings, + /// Image generation. + ImageGeneration, + /// Audio input or output. + Audio, + /// Asynchronous batch execution. + Batch, +} + +/// Explicit provider support for non-text execution surfaces. +/// +/// These flags are deliberately separate from the existing text/image/tool +/// fields. A provider must opt into each surface only after its adapter, +/// bounds, accounting, and parity fixtures are complete. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct ModalityCapabilities { + /// Vector embeddings are supported. + pub embeddings: bool, + /// Image generation is supported. + pub image_generation: bool, + /// Audio input is supported. + pub audio_input: bool, + /// Audio output is supported. + pub audio_output: bool, + /// Durable batch execution is supported. + pub batch: bool, +} + /// Capabilities that can make a destination ineligible before scoring. #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct DestinationCapabilities { @@ -105,6 +142,9 @@ pub struct DestinationCapabilities { pub image_input: bool, /// Reviewed tool calls are supported. pub tools: bool, + /// Explicit non-text surface capabilities. + #[serde(default)] + pub modalities: ModalityCapabilities, } /// A host-provided, secret-free snapshot of one concrete destination. @@ -145,6 +185,9 @@ pub struct RoutingRequirements { pub tools: bool, /// The caller requires incremental output. pub streaming: bool, + /// Surface requested by the caller; defaults to text for compatibility. + #[serde(default)] + pub surface: InferenceSurface, } /// Platform-neutral input to the authoritative routing engine. @@ -182,6 +225,14 @@ pub enum ExclusionReason { ToolsUnsupported, /// Streaming is required but unsupported. StreamingUnsupported, + /// Embeddings are required but unsupported. + EmbeddingsUnsupported, + /// Image generation is required but unsupported. + ImageGenerationUnsupported, + /// Audio is required but unsupported. + AudioUnsupported, + /// Batch execution is required but unsupported. + BatchUnsupported, /// The user has excluded this destination from Automatic. AutomaticNotAllowed, } From 85ea98ba0f37054637e4dbd89dc90511e07bc424 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Mon, 3 Aug 2026 05:44:17 +0000 Subject: [PATCH 2/2] fix(apple): update Swift callers for modality contracts [roadmap:WF-ROADMAP-0010] The bounded modality contracts added five capability flags to DestinationCapabilities and a surface field to RoutingRequirements, but the hand-written Swift callers were not updated, so the Apple bridge tests and the iOS shell no longer compile against the generated bridge. Update every construction site to the new signatures, defaulting each non-text capability to false and each request surface to .text, matching the fail-closed defaults in wayfinder-apple-ffi. --- .../WayfinderRoutingBridgeTests.swift | 13 ++++++++++--- ios/WayfinderIOS/WayfinderIOS/AppModel.swift | 17 ++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/apple/Packages/WayfinderRoutingBridge/Tests/WayfinderRoutingBridgeTests/WayfinderRoutingBridgeTests.swift b/apple/Packages/WayfinderRoutingBridge/Tests/WayfinderRoutingBridgeTests/WayfinderRoutingBridgeTests.swift index 935d6fa..2fc92c4 100644 --- a/apple/Packages/WayfinderRoutingBridge/Tests/WayfinderRoutingBridgeTests/WayfinderRoutingBridgeTests.swift +++ b/apple/Packages/WayfinderRoutingBridge/Tests/WayfinderRoutingBridgeTests/WayfinderRoutingBridgeTests.swift @@ -31,7 +31,12 @@ final class WayfinderRoutingBridgeTests: XCTestCase { text: true, streaming: true, imageInput: false, - tools: false + tools: false, + embeddings: false, + imageGeneration: false, + audioInput: false, + audioOutput: false, + batch: false ), automaticEligible: true ) @@ -69,7 +74,8 @@ final class WayfinderRoutingBridgeTests: XCTestCase { contextTokens: 1_024, imageInput: false, tools: false, - streaming: true + streaming: true, + surface: .text ) ), candidates: [ @@ -129,7 +135,8 @@ final class WayfinderRoutingBridgeTests: XCTestCase { contextTokens: nil, imageInput: false, tools: false, - streaming: false + streaming: false, + surface: .text ) ), candidates: [] diff --git a/ios/WayfinderIOS/WayfinderIOS/AppModel.swift b/ios/WayfinderIOS/WayfinderIOS/AppModel.swift index 0fc569a..1028bef 100644 --- a/ios/WayfinderIOS/WayfinderIOS/AppModel.swift +++ b/ios/WayfinderIOS/WayfinderIOS/AppModel.swift @@ -436,7 +436,8 @@ final class AppModel { contextTokens: nil, imageInput: false, tools: false, - streaming: true + streaming: true, + surface: .text ) ) let plan: RoutePlan @@ -1240,7 +1241,12 @@ struct RoutingDestination: Identifiable, Hashable { text: true, streaming: true, imageInput: false, - tools: false + tools: false, + embeddings: false, + imageGeneration: false, + audioInput: false, + audioOutput: false, + batch: false ), automaticEligible: automaticEligible ) @@ -1261,7 +1267,12 @@ struct RoutingDestination: Identifiable, Hashable { text: true, streaming: true, imageInput: false, - tools: false + tools: false, + embeddings: false, + imageGeneration: false, + audioInput: false, + audioOutput: false, + batch: false ), automaticEligible: true )