From 5ac480ec5b43b7b5743e41f45cd77541200db21a Mon Sep 17 00:00:00 2001 From: Andrei Hasna Date: Tue, 23 Jun 2026 00:13:01 +0300 Subject: [PATCH] feat: add multi-gateway smart routing --- .env.example | 14 + README.md | 28 +- docs/README.md | 14 +- docs/api-contract.md | 29 ++- docs/architecture.md | 12 + docs/provider-adapters.md | 61 +++++ docs/provider-references.md | 58 ++++- docs/routing-and-policy.md | 30 +++ docs/security-compliance.md | 14 + examples/cloudflare-ai-gateway/README.md | 29 +++ examples/helicone-ai-gateway/README.md | 24 ++ examples/kong-ai-gateway/README.md | 29 +++ examples/litellm-proxy/README.md | 29 +++ examples/openrouter-auto/README.md | 46 ++++ examples/portkey/README.md | 33 +++ examples/vercel-ai-gateway/README.md | 41 +++ gateway.config.example.json | 6 +- src/config.ts | 54 +++- src/gateway.ts | 40 ++- src/index.ts | 10 + src/presets.ts | 228 ++++++++++++++++- src/provider-config.ts | 77 ++++++ src/providers/openai-compatible.ts | 209 ++++++++++++++- src/router.ts | 313 +++++++++++++++++++++-- src/smoke.ts | 6 +- src/types.ts | 60 ++++- tests/config.test.ts | 142 ++++++++++ tests/gateway.test.ts | 74 ++++++ tests/provider.test.ts | 147 +++++++++++ tests/router.test.ts | 271 ++++++++++++++++++++ 30 files changed, 2053 insertions(+), 75 deletions(-) create mode 100644 examples/cloudflare-ai-gateway/README.md create mode 100644 examples/helicone-ai-gateway/README.md create mode 100644 examples/kong-ai-gateway/README.md create mode 100644 examples/litellm-proxy/README.md create mode 100644 examples/openrouter-auto/README.md create mode 100644 examples/portkey/README.md create mode 100644 examples/vercel-ai-gateway/README.md create mode 100644 src/provider-config.ts diff --git a/.env.example b/.env.example index 34126c3..95c8cfc 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,20 @@ ANTHROPIC_API_KEY= GOOGLE_GENERATIVE_AI_API_KEY= OPENROUTER_API_KEY= +# Gateway providers +AI_GATEWAY_API_KEY= +LITELLM_PROXY_BASE_URL= +LITELLM_API_KEY= +PORTKEY_API_KEY= +PORTKEY_CONFIG_ID= +PORTKEY_PROVIDER= +PORTKEY_VIRTUAL_KEY= +CLOUDFLARE_API_TOKEN= +CLOUDFLARE_AI_GATEWAY_BASE_URL= +HELICONE_API_KEY= +KONG_AI_GATEWAY_BASE_URL= +KONG_AI_GATEWAY_API_KEY= + # Chinese and China-focused providers DEEPSEEK_API_KEY= DASHSCOPE_API_KEY= diff --git a/README.md b/README.md index 0192b24..9d13577 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Hasna Gateway -Hasna Gateway is the open-source AI gateway core for Hasna apps and self-hosted teams. It exposes one stable OpenAI-compatible API while routing requests across providers, including OpenAI, Google Gemini, OpenRouter, DeepSeek, Qwen/DashScope, Kimi/Moonshot, Z.AI/GLM, and SiliconFlow. +Hasna Gateway is the open-source AI gateway core for Hasna apps and self-hosted teams. It exposes one stable OpenAI-compatible API while routing requests across providers, including OpenAI, Google Gemini, OpenRouter, Vercel AI Gateway, LiteLLM Proxy, Portkey, Cloudflare AI Gateway, Helicone AI Gateway, Kong AI Gateway, DeepSeek, Qwen/DashScope, Kimi/Moonshot, Z.AI/GLM, and SiliconFlow. The open-source package is useful on its own. Anyone can run it locally or on their own server, bring their own provider keys, define routing policy, and point applications at one endpoint. The hosted Hasna gateway can build on the same core while keeping accounts, billing, pooled provider contracts, discounts, tenant policy, and hosted observability private. @@ -9,7 +9,7 @@ The open-source package is useful on its own. Anyone can run it locally or on th - OpenAI-compatible HTTP API first, starting with `/v1/chat/completions`. - One gateway key for clients, many provider keys behind the gateway. - Bring-your-own-key mode for self-hosted users. -- Routing by model alias, provider allowlist/blocklist, region policy, price ceilings, fallback, and capability. +- Routing by model alias, provider allowlist/blocklist, region policy, price ceilings, fallback, capability, and smart cost/quality/latency hints. - Explicit China/provider policy so requests are never silently routed to a region or provider class the caller did not allow. - Usage normalization, estimated cost hooks, route decision metadata, and optional local JSONL usage ledger. - Hard or soft budgets by gateway key, tenant, and model alias across USD plus input/output/total tokens. @@ -88,6 +88,28 @@ Required config examples: Provider keys are loaded from environment variables only. Do not put provider secrets in config files. +Providers can use `baseUrl`, `baseUrlEnv`, `apiKeyEnv`, custom `auth`, and static or env-derived `headers`. This keeps OpenAI-compatible gateways on the generic adapter instead of adding hardcoded adapter forks. The built-in presets include: + +- Direct/provider presets: `openai`, `openrouter`, `deepseek`, `qwen`, `kimi`, `zai`, `siliconflow`. +- Gateway presets: `vercel-ai-gateway`, `litellm-proxy`, `portkey`, `cloudflare-ai-gateway`, `helicone-ai-gateway`, `kong-ai-gateway`. + +Smart routing is available with route mode `smart` or request `gateway.routing: "smart"`. It filters by policy first, then scores eligible candidates using configured prices, context, capabilities, quality/latency/success/throughput hints, and deterministic fallback ordering when metrics are missing. + +```json +{ + "model": "coding", + "messages": [{ "role": "user", "content": "Refactor this function." }], + "gateway": { + "routing": "smart", + "priority": "quality", + "cost_quality_tradeoff": 3, + "required_capabilities": ["tools", "json"], + "min_context_tokens": 128000, + "sticky_session_id": "thread-123" + } +} +``` + Budgets live in the same JSON config and spend is calculated from the usage ledger. JSONL append through `storage.usageLedgerPath` is the local-first default. Daily, monthly, and lifetime budgets require either `storage.usageLedgerPath` or an explicit `storage.cloud` backend; per-request budgets can run without cumulative storage. Use `mode: "hard"` to block exhausted budgets with an OpenAI-compatible `402` error, or `mode: "soft"` to keep serving while exposing warnings in gateway metadata and ledger records. ### Runtime Modes @@ -105,6 +127,8 @@ Set `runtime.mode` to `production-cloud` when running the gateway behind a cloud Production cloud mode does not create DNS, ACM, API Gateway, secrets, provider keys, or cloud infrastructure. Those deployment steps require an operator-owned deployment workflow outside this package. +The companion `open-router` repo is currently documented as the future extraction point for prompt-aware routing and eval harnesses. The deterministic routing implementation lives in this package today because it is tightly coupled to gateway policy, provider config, budgets, attempts, and ledger metadata. + ## Documentation - [Product requirements](docs/product-requirements.md) diff --git a/docs/README.md b/docs/README.md index df4fb3e..6a5ef00 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,9 +31,21 @@ The first implementation should prioritize a small working gateway over broad in - Config validation. - Model aliases. - Fallback routing. +- Smart cost/quality/latency routing. - Explicit provider policy. +- Config-driven provider auth and headers. - Streaming. - Usage normalization. - Tests. -Provider breadth should come after the request lifecycle is reliable. +Provider breadth should stay on the generic OpenAI-compatible adapter when the upstream gateway uses standard chat completions plus headers or documented request-body provider options. + +## Gateway Examples + +- [OpenRouter Auto Router](../examples/openrouter-auto/README.md) +- [Vercel AI Gateway](../examples/vercel-ai-gateway/README.md) +- [Portkey AI Gateway](../examples/portkey/README.md) +- [Cloudflare AI Gateway](../examples/cloudflare-ai-gateway/README.md) +- [LiteLLM Proxy](../examples/litellm-proxy/README.md) +- [Helicone AI Gateway](../examples/helicone-ai-gateway/README.md) +- [Kong AI Gateway](../examples/kong-ai-gateway/README.md) diff --git a/docs/api-contract.md b/docs/api-contract.md index 4118db7..11d9457 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -89,7 +89,11 @@ Example: ], "stream": true, "gateway": { - "routing": "fallback", + "routing": "smart", + "priority": "quality", + "cost_quality_tradeoff": 3, + "required_capabilities": ["tools"], + "min_context_tokens": 128000, "allowed_providers": ["deepseek", "qwen", "openai"], "blocked_regions": ["cn"], "max_output_usd_per_million_tokens": 10 @@ -97,7 +101,14 @@ Example: } ``` -The optional `gateway` field is a gateway-specific extension. It should be ignored before forwarding to providers. +The optional `gateway` field is a gateway-specific extension. It is ignored before forwarding to direct providers. For gateway providers with documented request-body routing controls, Hasna Gateway maps only supported fields: + +- OpenRouter: `provider.order`, `only`, `ignore`, `sort`, `max_price`, `allow_fallbacks`, `zdr`, `data_collection`, and Auto Router plugin options such as `allowed_models` and `cost_quality_tradeoff`. +- Vercel AI Gateway: `providerOptions.gateway.order`, `only`, `caching`, and `providerTimeouts`. + +Unsupported gateway-only fields and secrets are stripped. + +Smart routing fields include `task`, `priority`, `cost_quality_tradeoff`, `sticky_session_id`, `min_quality`, `min_context_tokens`, `expected_input_tokens`, `required_capabilities`, `provider_order`, `provider_only`, and `provider_ignore`. Policy is applied before scoring. ## Response Shape @@ -129,7 +140,19 @@ Non-streaming responses should match OpenAI chat completion shape: "provider_model": "deepseek-chat", "route_mode": "fallback", "attempts": 1, - "estimated_cost_usd": 0.00012 + "estimated_cost_usd": 0.00012, + "route_decision": { + "requested_model": "coding", + "selected": "deepseek/deepseek-chat", + "scores": [ + { + "provider": "deepseek", + "model": "deepseek/deepseek-chat", + "score": 0.82, + "reason": "highest cost, quality, latency, and success weighted score among eligible models" + } + ] + } } } ``` diff --git a/docs/architecture.md b/docs/architecture.md index 4454b7f..e8f6794 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,9 +59,12 @@ The router receives a normalized request and eligible model candidates. It choos - `lowest-latency`: choose lowest recent p95 or configured latency. - `highest-throughput`: choose provider with best recent success and throughput. - `balanced`: weighted score from cost, latency, success rate, and quality hints. +- `smart`: request-tunable scoring across cost, quality, latency, success, context, and capability hints. Routing should always produce a route decision object that can be logged and tested. +Policy filtering happens before scoring. The router must not let a high score route outside region, data, BYOK, credential, capability, or cost policy. + ### Provider Adapter Layer Adapters convert the internal request to provider-specific requests and normalize responses back into the gateway response shape. A provider adapter owns: @@ -130,6 +133,15 @@ Recommended internal modules: - `src/errors`: provider error taxonomy. - `src/sdk`: embeddable TypeScript API. +## open-router Companion + +The companion `open-router` repository is intended for reusable prompt-aware routing, provider scoring, and evaluation harnesses. It is currently a companion placeholder, not a package dependency. + +The deterministic smart routing layer belongs in `open-gateway` for now because it depends directly on gateway config, data policy, credentials, route metadata, budgets, usage ledger records, and provider attempt accounting. Once `open-router` has reusable package code, the boundary should be: + +- `open-gateway`: policy, credentials, budgets, provider attempts, ledger, and OpenAI-compatible HTTP surface. +- `open-router`: optional prompt-aware scoring/evals that receive already-policy-filtered candidates and return explainable ranking metadata. + ## Existing Hasna Code To Reuse - Provider conversion ideas from `open-aicopilot`. diff --git a/docs/provider-adapters.md b/docs/provider-adapters.md index d403f43..77fc0e2 100644 --- a/docs/provider-adapters.md +++ b/docs/provider-adapters.md @@ -46,6 +46,67 @@ The built-in `google` preset is intentionally conservative about data policy. Op OpenRouter can be supported as a provider adapter and as a routing backend. It should not be the only gateway strategy. Hasna Gateway should still be able to call providers directly. +The current implementation keeps OpenRouter on the OpenAI-compatible adapter and maps only documented request body controls to `provider`, `plugins`, and `session_id`. This avoids an adapter fork while still supporting provider selection, ZDR, data collection, max price, and Auto Router options. + +## Generic Gateway Provider Config + +OpenAI-compatible providers and gateways can be configured without code changes: + +```json +{ + "id": "example-gateway", + "displayName": "Example Gateway", + "kind": "openai-compatible", + "baseUrlEnv": "EXAMPLE_GATEWAY_BASE_URL", + "auth": { + "type": "header", + "apiKeyEnv": "EXAMPLE_GATEWAY_KEY", + "headerName": "x-api-key", + "prefix": "" + }, + "headers": { + "x-config-id": { "env": "EXAMPLE_GATEWAY_CONFIG_ID" }, + "x-static": "static-value" + }, + "dataPolicy": { + "allowTraining": false, + "allowLogging": false, + "byokOnly": true + } +} +``` + +Supported provider config fields: + +- `baseUrl`: static OpenAI-compatible base URL. +- `baseUrlEnv`: environment variable containing the base URL. +- `apiKeyEnv`: shorthand for bearer auth. +- `auth.type`: `bearer`, `header`, or `none`. +- `auth.apiKeyEnv`, `auth.headerName`, `auth.prefix`: custom credential header settings. +- `headers`: static values or `{ "env": "...", "prefix": "...", "required": true }`. + +Built-in gateway presets: + +- `vercel-ai-gateway` +- `litellm-proxy` +- `portkey` +- `cloudflare-ai-gateway` +- `helicone-ai-gateway` +- `kong-ai-gateway` + +These remain normal route candidates. If the upstream gateway performs its own fallback or load balancing, Hasna Gateway records that upstream as one provider attempt unless the route config lists additional Hasna candidates. + +## Provider Option Mapping + +Direct providers receive only OpenAI-compatible request fields. Gateway-specific fields are stripped. + +Mapped gateway bodies: + +- OpenRouter: `provider.order`, `only`, `ignore`, `sort`, `max_price`, `allow_fallbacks`, `zdr`, `data_collection`, `quantizations`, `preferred_min_throughput`, and Auto Router plugin `allowed_models` / `cost_quality_tradeoff`. +- Vercel AI Gateway: `providerOptions.gateway.order`, `only`, `caching`, and `providerTimeouts`. + +Portkey, Cloudflare, LiteLLM, Helicone, and Kong are supported through provider config headers/auth and OpenAI-compatible model IDs. Their own routing/load-balancing configs stay in those systems. + ## Chinese Provider Priority These providers should be first-class because they are important for cost, coding, and international model access: diff --git a/docs/provider-references.md b/docs/provider-references.md index f55813c..97f4328 100644 --- a/docs/provider-references.md +++ b/docs/provider-references.md @@ -1,6 +1,6 @@ # 2026 Provider References -These notes capture provider docs checked during project setup and rechecked during implementation on 2026-06-16. Model names change quickly, so release smoke tests should still prefer provider `/models` APIs when credentials are available. +These notes capture provider docs checked during project setup, rechecked during implementation on 2026-06-16, and rechecked for multi-gateway routing on 2026-06-22. Model names change quickly, so release smoke tests should still prefer provider `/models` APIs when credentials are available. ## DeepSeek @@ -41,3 +41,59 @@ These notes capture provider docs checked during project setup and rechecked dur - Auth: `Authorization: Bearer ` - Example model: `Pro/zai-org/GLM-4.7` - Usage includes normal OpenAI token fields plus reasoning and cache details on some models. + +## OpenRouter + +- Docs: https://openrouter.ai/docs/guides/routing/provider-selection +- Auto Router docs: https://openrouter.ai/docs/guides/routing/routers/auto-router +- OpenAI-compatible base URL: `https://openrouter.ai/api/v1` +- Provider routing uses a `provider` object with fields such as `order`, `only`, `ignore`, `sort`, `max_price`, `allow_fallbacks`, `zdr`, and `data_collection`. +- Auto Router uses model `openrouter/auto`; per-request Auto Router settings use an `auto-router` plugin with `allowed_models` and `cost_quality_tradeoff`. +- Hasna Gateway maps only these documented fields and strips other gateway-only fields for direct providers. + +## Vercel AI Gateway + +- Docs: https://vercel.com/docs/ai-gateway/models-and-providers/provider-options +- OpenAI-compatible base URL: `https://ai-gateway.vercel.sh/v1` +- Provider options use `providerOptions.gateway` with `order`, `only`, `caching`, and `providerTimeouts`. +- BYOK credentials are managed in Vercel AI Gateway settings; requests should not include provider secrets through Hasna Gateway. + +## LiteLLM Proxy + +- Docs: https://docs.litellm.ai/docs/routing +- Proxy load balancing docs: https://docs.litellm.ai/docs/proxy/load_balancing +- OpenAI-compatible proxy base URL is deployment-specific, commonly `http://127.0.0.1:4000/v1`. +- LiteLLM owns its internal routing strategies such as weighted pick, latency-based, cost-based, and order fallback. Hasna Gateway treats the LiteLLM proxy as one upstream candidate unless route config adds additional candidates. + +## Portkey AI Gateway + +- Config docs: https://portkey.ai/docs/product/ai-gateway/configs +- Load balancing docs: https://portkey.ai/docs/product/ai-gateway/load-balancing +- OpenAI-compatible gateway URL: `https://api.portkey.ai/v1` +- Gateway config selection can be passed with `x-portkey-config`; generic header auth supports `x-portkey-api-key` and optional provider/virtual-key headers. + +## Cloudflare AI Gateway + +- REST API docs: https://developers.cloudflare.com/ai-gateway/usage/rest-api/ +- OpenAI-compatible REST base URL: `https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/ai/v1` +- Deprecated compat base URL: `https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/compat` +- Auth uses a Cloudflare API token in `Authorization`. + +## Helicone AI Gateway + +- Docs: https://docs.helicone.ai/gateway/overview +- Integration docs: https://docs.helicone.ai/gateway/integrations/overview +- OpenAI-compatible base URL: `https://ai-gateway.helicone.ai/v1` +- The preset uses `Helicone-Auth: Bearer `. + +## Kong AI Gateway + +- Docs: https://developer.konghq.com/ai-gateway/ +- Load balancing docs: https://developer.konghq.com/ai-gateway/load-balancing/ +- Base URL and auth depend on the deployed Kong route and plugins. +- Kong can perform its own load balancing, retries, fallback, and semantic routing. Hasna Gateway records Kong as one upstream attempt unless configured with additional local fallback candidates. + +## RouteLLM + +- Repo: https://github.com/lm-sys/routellm +- RouteLLM is useful for prompt-aware routing and evaluations, but it is not embedded in this implementation. The current smart routing is deterministic and config-driven inside Hasna Gateway. The `open-router` companion repo is the future extraction point for prompt-aware/eval routing when reusable package code exists. diff --git a/docs/routing-and-policy.md b/docs/routing-and-policy.md index 98ddfea..a3f394f 100644 --- a/docs/routing-and-policy.md +++ b/docs/routing-and-policy.md @@ -11,6 +11,7 @@ The default self-hosted behavior should be conservative: - Do not call a provider without a configured key. - Do not call a hosted Hasna endpoint unless the user configured it. - Do not route to China-region or China-owned providers unless the route or config allows them. +- Do not score or prefer a candidate until region, data, BYOK, credential, capability, and cost policy filters have passed. Production cloud behavior must be explicit instead of inferred from deployment context. A config with `runtime.mode: "production-cloud"` should bind to a non-loopback interface, keep gateway auth required, require runtime secrets and route readiness for `/health`, and constrain provider discovery to HTTPS provider URLs plus any exact origins listed in `runtime.serviceDiscovery.allowedProviderBaseUrls`. @@ -115,6 +116,35 @@ Fallbacks should not hide: - User input errors. - Unsafe region or data policy mismatches. +## Smart Routing + +Smart routing is an ordered policy and scoring layer: + +1. Resolve the requested model or alias into configured candidates. +2. Apply policy filters first: provider allow/block lists, region, China opt-in, data retention, training/logging, BYOK, credentials, model capability, context window, and price ceilings. +3. Score only the remaining eligible candidates. +4. Return the route decision, skipped reasons, scores, and selected model in gateway metadata and ledger records. + +Supported route modes: + +- `fallback`: first eligible candidate in configured order. +- `cheapest`: lowest configured input plus output token price. If no eligible candidate has prices, the route fails closed. +- `lowest-latency`: latency-weighted score using configured `averageLatencyMs` when present. +- `highest-throughput`: throughput and success weighted score using configured `throughputTokensPerSecond` and `successRate`. +- `balanced`: weighted score across cost, quality, latency, and success. +- `smart`: same score inputs as `balanced`, adjusted by request hints such as `priority` and `cost_quality_tradeoff`. + +Request hints under `gateway` can reduce the eligible set or tune scoring: + +- `priority`: `cost`, `quality`, `latency`, or `balanced`. +- `cost_quality_tradeoff`: `0` favors quality, `10` favors cost. +- `sticky_session_id` or `session_id`: deterministic tie-breaking for repeated conversations. +- `required_capabilities`: capabilities such as `tools`, `json`, `vision`, or `reasoning`. +- `min_quality` and `min_context_tokens`. +- `provider_order`, `provider_only`, and `provider_ignore`. + +When configured metrics are missing, smart routing uses deterministic fallback values and original candidate order. It does not read the usage ledger inside synchronous route resolution; future runtime metric injection can pass precomputed latency/success data into routing without changing the fail-closed policy order. + ## Cost Controls Cost controls should support: diff --git a/docs/security-compliance.md b/docs/security-compliance.md index 69c2b8a..a01b432 100644 --- a/docs/security-compliance.md +++ b/docs/security-compliance.md @@ -29,6 +29,18 @@ Recommended open-source env variables: - `ANTHROPIC_API_KEY` - `GOOGLE_GENERATIVE_AI_API_KEY` - `OPENROUTER_API_KEY` +- `AI_GATEWAY_API_KEY` +- `LITELLM_PROXY_BASE_URL` +- `LITELLM_API_KEY` +- `PORTKEY_API_KEY` +- `PORTKEY_CONFIG_ID` +- `PORTKEY_PROVIDER` +- `PORTKEY_VIRTUAL_KEY` +- `CLOUDFLARE_API_TOKEN` +- `CLOUDFLARE_AI_GATEWAY_BASE_URL` +- `HELICONE_API_KEY` +- `KONG_AI_GATEWAY_BASE_URL` +- `KONG_AI_GATEWAY_API_KEY` - `DEEPSEEK_API_KEY` - `DASHSCOPE_API_KEY` - `MOONSHOT_API_KEY` @@ -79,6 +91,8 @@ If provider terms, retention, or region are unknown, the gateway should treat th Provider service discovery must be explicit for production cloud runtime. Enabled providers must use configured `baseUrl` values, and operators can set `runtime.serviceDiscovery.allowedProviderBaseUrls` to a list of exact provider origins. Local/private endpoints and non-HTTPS provider URLs are rejected by default in production cloud mode. These are static config checks; DNS resolution and cloud egress policy remain operator responsibilities. +Gateway providers such as Portkey, Cloudflare, Vercel, Helicone, Kong, and LiteLLM may perform their own logging, routing, fallback, billing, or retention. Configure their `dataPolicy` conservatively and only enable them on routes whose logging, region, and BYOK requirements they can satisfy. + ## Abuse Controls Open-source self-hosted mode should include basic controls: diff --git a/examples/cloudflare-ai-gateway/README.md b/examples/cloudflare-ai-gateway/README.md new file mode 100644 index 0000000..b88d1da --- /dev/null +++ b/examples/cloudflare-ai-gateway/README.md @@ -0,0 +1,29 @@ +# Cloudflare AI Gateway + +Use the `cloudflare-ai-gateway` preset with an account-specific base URL. + +```bash +CLOUDFLARE_API_TOKEN= +CLOUDFLARE_AI_GATEWAY_BASE_URL=https://api.cloudflare.com/client/v4/accounts//ai/v1 +``` + +```json +{ + "presets": ["cloudflare-ai-gateway"], + "routes": [ + { + "id": "cloudflare-coding", + "mode": "fallback", + "modelAliases": ["cloudflare-coding"], + "fallbackModelIds": ["cloudflare-ai-gateway/openai/gpt-4.1-mini"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": true, + "allowedRegions": ["global"] + } + } + ] +} +``` + +Cloudflare's current REST API exposes an OpenAI-compatible chat completions path under `/ai/v1`; the gateway adapter appends `/chat/completions`. diff --git a/examples/helicone-ai-gateway/README.md b/examples/helicone-ai-gateway/README.md new file mode 100644 index 0000000..9fcdc5f --- /dev/null +++ b/examples/helicone-ai-gateway/README.md @@ -0,0 +1,24 @@ +# Helicone AI Gateway + +Use the `helicone-ai-gateway` preset when Helicone owns the upstream gateway and observability layer. + +```json +{ + "presets": ["helicone-ai-gateway"], + "routes": [ + { + "id": "helicone-coding", + "mode": "fallback", + "modelAliases": ["helicone-coding"], + "fallbackModelIds": ["helicone-ai-gateway/openai/gpt-4.1-mini"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": true, + "allowedRegions": ["global"] + } + } + ] +} +``` + +Set `HELICONE_API_KEY`. The preset uses generic header auth through `Helicone-Auth: Bearer `. diff --git a/examples/kong-ai-gateway/README.md b/examples/kong-ai-gateway/README.md new file mode 100644 index 0000000..65c8dd2 --- /dev/null +++ b/examples/kong-ai-gateway/README.md @@ -0,0 +1,29 @@ +# Kong AI Gateway + +Use the `kong-ai-gateway` preset for a self-hosted Kong AI Gateway or Kong route that exposes an OpenAI-compatible endpoint. + +```bash +KONG_AI_GATEWAY_BASE_URL=https://kong.example.com/v1 +KONG_AI_GATEWAY_API_KEY= +``` + +```json +{ + "presets": ["kong-ai-gateway"], + "routes": [ + { + "id": "kong-coding", + "mode": "fallback", + "modelAliases": ["kong-coding"], + "fallbackModelIds": ["kong-ai-gateway/coding"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": false, + "allowedRegions": ["private"] + } + } + ] +} +``` + +Kong can perform its own load balancing and semantic routing. Hasna Gateway still records the Kong route as one upstream attempt. diff --git a/examples/litellm-proxy/README.md b/examples/litellm-proxy/README.md new file mode 100644 index 0000000..8ec757d --- /dev/null +++ b/examples/litellm-proxy/README.md @@ -0,0 +1,29 @@ +# LiteLLM Proxy + +Use the `litellm-proxy` preset when LiteLLM owns an internal model group and Hasna Gateway owns external policy, budgets, and metadata. + +```bash +LITELLM_PROXY_BASE_URL=http://127.0.0.1:4000/v1 +LITELLM_API_KEY= +``` + +```json +{ + "presets": ["litellm-proxy"], + "routes": [ + { + "id": "litellm-coding", + "mode": "fallback", + "modelAliases": ["litellm-coding"], + "fallbackModelIds": ["litellm-proxy/coding"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": false, + "allowedRegions": ["private"] + } + } + ] +} +``` + +Keep LiteLLM routing details in LiteLLM config. Hasna Gateway treats the proxy as one upstream candidate. diff --git a/examples/openrouter-auto/README.md b/examples/openrouter-auto/README.md new file mode 100644 index 0000000..be58037 --- /dev/null +++ b/examples/openrouter-auto/README.md @@ -0,0 +1,46 @@ +# OpenRouter Auto Router + +Use the built-in `openrouter` preset with the `openrouter/auto` model preset when you want OpenRouter to choose the underlying model. Hasna Gateway still applies local policy before calling OpenRouter. + +```json +{ + "presets": ["openrouter"], + "routes": [ + { + "id": "openrouter-auto", + "mode": "fallback", + "modelAliases": ["gateway-auto"], + "fallbackModelIds": ["openrouter/auto"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": false, + "allowedRegions": ["global"] + } + } + ] +} +``` + +Request with provider routing and Auto Router options: + +```json +{ + "model": "gateway-auto", + "messages": [{ "role": "user", "content": "Pick the right model for this task." }], + "gateway": { + "provider_order": ["anthropic", "openai"], + "provider_only": ["anthropic", "openai"], + "allow_fallbacks": true, + "zero_data_retention_required": true, + "cost_quality_tradeoff": 3, + "sticky_session_id": "conversation-123" + }, + "provider_options": { + "openrouter": { + "allowed_models": ["anthropic/*", "openai/gpt-5*"] + } + } +} +``` + +Set `OPENROUTER_API_KEY`. The preset sends OpenRouter attribution headers through generic provider headers. diff --git a/examples/portkey/README.md b/examples/portkey/README.md new file mode 100644 index 0000000..cc1817d --- /dev/null +++ b/examples/portkey/README.md @@ -0,0 +1,33 @@ +# Portkey AI Gateway + +Use the `portkey` preset when you want Hasna Gateway to call a Portkey gateway config as an OpenAI-compatible upstream. + +```json +{ + "presets": ["portkey"], + "routes": [ + { + "id": "portkey-coding", + "mode": "fallback", + "modelAliases": ["portkey-coding"], + "fallbackModelIds": ["portkey/openai/gpt-4.1-mini"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": true, + "allowedRegions": ["global"] + } + } + ] +} +``` + +Set these environment variables as needed: + +```bash +PORTKEY_API_KEY= +PORTKEY_CONFIG_ID= +PORTKEY_PROVIDER= +PORTKEY_VIRTUAL_KEY= +``` + +The preset uses generic header auth: `x-portkey-api-key`, plus optional config/provider headers. diff --git a/examples/vercel-ai-gateway/README.md b/examples/vercel-ai-gateway/README.md new file mode 100644 index 0000000..95f1691 --- /dev/null +++ b/examples/vercel-ai-gateway/README.md @@ -0,0 +1,41 @@ +# Vercel AI Gateway + +Use the `vercel-ai-gateway` preset for Vercel's OpenAI-compatible gateway endpoint. + +```json +{ + "presets": ["vercel-ai-gateway"], + "routes": [ + { + "id": "vercel-coding", + "mode": "fallback", + "modelAliases": ["vercel-coding"], + "fallbackModelIds": ["vercel-ai-gateway/openai/gpt-4.1-mini"], + "dataPolicy": { + "allowTraining": false, + "allowLogging": true, + "allowedRegions": ["global"] + } + } + ] +} +``` + +Request with Vercel gateway provider options: + +```json +{ + "model": "vercel-coding", + "messages": [{ "role": "user", "content": "Implement a small TypeScript helper." }], + "gateway": { + "provider_order": ["bedrock", "anthropic", "openai"], + "provider_only": ["bedrock", "anthropic", "openai"], + "caching": "auto", + "provider_timeouts": { + "byok": { "anthropic": 3000, "openai": 5000 } + } + } +} +``` + +Set `AI_GATEWAY_API_KEY`. Keep provider credentials in Vercel's gateway settings when using BYOK. diff --git a/gateway.config.example.json b/gateway.config.example.json index 59b07ed..5ae4103 100644 --- a/gateway.config.example.json +++ b/gateway.config.example.json @@ -51,7 +51,7 @@ "deepseek/deepseek-v4-pro", "qwen/qwen3-coder-plus", "kimi/kimi-k2.7-code", - "zai/glm-5.1" + "zai/glm-5.2" ], "dataPolicy": { "allowTraining": false, @@ -80,12 +80,12 @@ "id": "china-coding", "mode": "fallback", "modelAliases": ["china-coding"], - "providerAllowlist": ["deepseek", "qwen", "kimi"], + "providerAllowlist": ["deepseek", "qwen", "kimi", "zai"], "fallbackModelIds": [ "deepseek/deepseek-v4-pro", "qwen/qwen3-coder-plus", "kimi/kimi-k2.7-code", - "zai/glm-5.1" + "zai/glm-5.2" ], "dataPolicy": { "allowTraining": false, diff --git a/src/config.ts b/src/config.ts index e49cb2b..702caa8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,6 @@ import { GatewayHttpError } from "./errors"; import { modelPresets, providerPresets } from "./presets"; +import { providerCredentialEnv, providerRequiresCredential } from "./provider-config"; import { resolveRoute } from "./router"; import { z } from "zod"; import type { @@ -137,6 +138,27 @@ const budgetSchema = z }) .passthrough(); +const providerAuthSchema = z + .object({ + type: z.enum(["bearer", "header", "none"]).optional(), + apiKeyEnv: z.string().min(1).optional(), + headerName: z.string().min(1).optional(), + prefix: z.string().optional(), + }) + .passthrough(); + +const providerHeaderValueSchema = z.union([ + z.string(), + z + .object({ + value: z.string().optional(), + env: z.string().min(1).optional(), + prefix: z.string().optional(), + required: z.boolean().optional(), + }) + .passthrough(), +]); + const providerSchema = z .object({ id: z.string().min(1), @@ -145,7 +167,10 @@ const providerSchema = z .enum(["openai-compatible", "openai", "anthropic", "google", "bedrock", "vertex", "openrouter"]) .default("openai-compatible"), baseUrl: z.string().url().optional(), + baseUrlEnv: z.string().min(1).optional(), apiKeyEnv: z.string().min(1).optional(), + auth: providerAuthSchema.optional(), + headers: z.record(providerHeaderValueSchema).optional(), enabled: z.boolean().optional(), regions: z.array(z.string().min(1)).optional(), jurisdiction: z.string().min(1).optional(), @@ -165,13 +190,17 @@ const modelSchema = z contextWindow: z.number().int().min(1).optional(), inputUsdPerMillionTokens: z.number().min(0).optional(), outputUsdPerMillionTokens: z.number().min(0).optional(), + qualityScore: z.number().min(0).max(1).optional(), + averageLatencyMs: z.number().min(1).optional(), + successRate: z.number().min(0).max(1).optional(), + throughputTokensPerSecond: z.number().min(0).optional(), }) .passthrough(); const routeSchema = z .object({ id: z.string().min(1), - mode: z.enum(["explicit", "fallback", "cheapest", "lowest-latency", "highest-throughput", "balanced"]), + mode: z.enum(["explicit", "fallback", "cheapest", "lowest-latency", "highest-throughput", "balanced", "smart"]), modelAliases: z.array(z.string().min(1)).optional(), providerAllowlist: z.array(z.string().min(1)).optional(), providerBlocklist: z.array(z.string().min(1)).optional(), @@ -601,14 +630,19 @@ export function validateConfig(input: GatewayConfigInput): GatewayConfigValidati assertString(provider.id, "provider.id", errors); assertString(provider.displayName, `provider ${provider.id}.displayName`, errors); assertString(provider.kind, `provider ${provider.id}.kind`, errors); - if (!provider.baseUrl) { - errors.push(`provider ${provider.id} must define baseUrl.`); + if (!provider.baseUrl && !provider.baseUrlEnv) { + errors.push(`provider ${provider.id} must define baseUrl or baseUrlEnv.`); } - if (!provider.apiKeyEnv) { - warnings.push(`provider ${provider.id} does not define apiKeyEnv and will not be callable.`); + if (providerRequiresCredential(provider) && !providerCredentialEnv(provider)) { + warnings.push(`provider ${provider.id} does not define apiKeyEnv/auth.apiKeyEnv and will not be callable.`); } - if (config.runtime.mode === "production-cloud" && provider.enabled !== false && !provider.apiKeyEnv) { - errors.push(`provider ${provider.id} must define apiKeyEnv in production-cloud mode.`); + if ( + config.runtime.mode === "production-cloud" && + provider.enabled !== false && + providerRequiresCredential(provider) && + !providerCredentialEnv(provider) + ) { + errors.push(`provider ${provider.id} must define apiKeyEnv or auth.apiKeyEnv in production-cloud mode.`); } if (config.runtime.mode === "production-cloud") { validateProductionProviderBoundary(config, provider, errors); @@ -780,8 +814,10 @@ export function validateRuntimeSecrets(config: GatewayConfig, env: Record { - if (provider.enabled === false || !provider.apiKeyEnv) return false; - return Boolean(env[provider.apiKeyEnv]); + if (provider.enabled === false) return false; + const credentialEnv = providerCredentialEnv(provider); + if (!providerRequiresCredential(provider)) return true; + return Boolean(credentialEnv && env[credentialEnv]); }); if (!hasCallableProvider) { diff --git a/src/gateway.ts b/src/gateway.ts index 4e2d4ed..70f76ff 100644 --- a/src/gateway.ts +++ b/src/gateway.ts @@ -9,6 +9,7 @@ import { spendFromUsage, } from "./budget"; import { adapterForProvider } from "./providers"; +import { providerCredentialEnv, providerRequiresCredential } from "./provider-config"; import { resolveRoute } from "./router"; import { transformOpenAICompatibleStream } from "./streaming"; import type { @@ -187,6 +188,27 @@ function requestWithStreamingUsage(request: OpenAIChatCompletionRequest): OpenAI }; } +function requestWithEffectivePolicy( + request: OpenAIChatCompletionRequest, + decision: GatewayRouteDecision, +): OpenAIChatCompletionRequest { + return { + ...request, + gateway: { + ...(request.gateway ?? {}), + allow_training: decision.policy.allow_training, + allow_logging: decision.policy.allow_logging, + allow_chinese_providers: decision.policy.allow_chinese_providers, + zero_data_retention_required: decision.policy.zero_data_retention_required, + byok_only: decision.policy.byok_only, + ...(decision.policy.allowed_providers ? { allowed_providers: decision.policy.allowed_providers } : {}), + ...(decision.policy.blocked_providers ? { blocked_providers: decision.policy.blocked_providers } : {}), + ...(decision.policy.allowed_regions ? { allowed_regions: decision.policy.allowed_regions } : {}), + ...(decision.policy.blocked_regions ? { blocked_regions: decision.policy.blocked_regions } : {}), + }, + }; +} + function extractProviderMessage(payload: unknown): string | undefined { if (!payload || typeof payload !== "object") return undefined; const record = payload as Record; @@ -214,9 +236,9 @@ async function parseProviderJson(response: Response): Promise): string { - const apiKeyEnv = candidate.provider.apiKeyEnv; + const apiKeyEnv = providerCredentialEnv(candidate.provider); const apiKey = apiKeyEnv ? env[apiKeyEnv] : undefined; - if (!apiKey) { + if (providerRequiresCredential(candidate.provider) && !apiKey) { throw new GatewayHttpError({ status: 400, type: "gateway_config_error", @@ -224,21 +246,23 @@ function apiKeyFor(candidate: GatewayRouteCandidate, env: Record { const adapter = adapterForProvider(candidate.provider); return adapter.send({ provider: candidate.provider, model: candidate.model, - request, + request: requestWithEffectivePolicy(request, decision), apiKey: apiKeyFor(candidate, options.env ?? process.env), timeoutMs: options.config.server.requestTimeoutMs, + env: options.env ?? process.env, fetchImpl: options.fetchImpl, }); } @@ -247,14 +271,16 @@ async function openProviderStream( options: GatewayRuntimeOptions, request: OpenAIChatCompletionRequest, candidate: GatewayRouteCandidate, + decision: GatewayRouteDecision, ): Promise { const adapter = adapterForProvider(candidate.provider); return adapter.stream({ provider: candidate.provider, model: candidate.model, - request, + request: requestWithEffectivePolicy(request, decision), apiKey: apiKeyFor(candidate, options.env ?? process.env), timeoutMs: options.config.server.requestTimeoutMs, + env: options.env ?? process.env, fetchImpl: options.fetchImpl, }); } @@ -299,7 +325,7 @@ export async function createChatCompletion( return cachedResult; } - const response = await callProvider(options, request, candidate); + const response = await callProvider(options, request, candidate, route.decision); const latencyMs = Date.now() - started; if (!response.ok) { @@ -445,7 +471,7 @@ export async function createChatCompletionStream( ? requestWithStreamingUsage(request) : request; hardBudgetRequiresUsage = budgetStatuses.some((status) => status.budget.mode === "hard"); - response = await openProviderStream(options, budgetedRequest, candidate); + response = await openProviderStream(options, budgetedRequest, candidate, route.decision); } catch (error) { lastError = error instanceof GatewayHttpError diff --git a/src/index.ts b/src/index.ts index 0864591..e1066fa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,13 @@ export { GatewayHttpError, gatewayErrorResponse, jsonError } from "./errors"; export { createChatCompletion, createChatCompletionStream } from "./gateway"; export { appendUsageLedger } from "./ledger"; export { toCapabilityCard, toCapabilityCards, toCostEstimate, toDecisionEnvelope } from "./lib/contracts"; +export { + buildProviderHeaders, + missingRequiredProviderHeaderEnvs, + providerBaseUrl, + providerCredentialEnv, + providerRequiresCredential, +} from "./provider-config"; export { modelPresets, providerPresets } from "./presets"; export { resolveRoute } from "./router"; export { createGatewayHandler, startGatewayServer } from "./server"; @@ -40,6 +47,8 @@ export type { GatewayModelCapability, GatewayModelConfig, GatewayProviderConfig, + GatewayProviderAuthConfig, + GatewayProviderHeaderValue, GatewayProviderKind, GatewayRateLimitConfig, GatewayRequestOptions, @@ -47,6 +56,7 @@ export type { GatewayRouteAttempt, GatewayRouteCandidate, GatewayRouteDecision, + GatewayRouteScore, GatewayRoutePolicy, GatewayRoutingMode, GatewayRuntimeOptions, diff --git a/src/presets.ts b/src/presets.ts index 524a544..9db8750 100644 --- a/src/presets.ts +++ b/src/presets.ts @@ -25,6 +25,10 @@ export const providerPresets: Record = { kind: "openai-compatible", baseUrl: "https://openrouter.ai/api/v1", apiKeyEnv: "OPENROUTER_API_KEY", + headers: { + "http-referer": "https://github.com/hasna/open-gateway", + "x-title": "Hasna Gateway", + }, enabled: true, regions: ["global"], dataPolicy: { @@ -49,6 +53,113 @@ export const providerPresets: Record = { byokOnly: true, }, }, + "vercel-ai-gateway": { + id: "vercel-ai-gateway", + displayName: "Vercel AI Gateway", + kind: "openai-compatible", + baseUrl: "https://ai-gateway.vercel.sh/v1", + apiKeyEnv: "AI_GATEWAY_API_KEY", + enabled: true, + regions: ["global"], + dataPolicy: { + allowTraining: false, + allowLogging: true, + zeroDataRetentionAvailable: false, + byokOnly: true, + }, + }, + "litellm-proxy": { + id: "litellm-proxy", + displayName: "LiteLLM Proxy", + kind: "openai-compatible", + baseUrlEnv: "LITELLM_PROXY_BASE_URL", + apiKeyEnv: "LITELLM_API_KEY", + enabled: true, + regions: ["private"], + jurisdiction: "self-hosted", + dataPolicy: { + allowTraining: false, + allowLogging: false, + zeroDataRetentionAvailable: true, + byokOnly: true, + }, + }, + portkey: { + id: "portkey", + displayName: "Portkey AI Gateway", + kind: "openai-compatible", + baseUrl: "https://api.portkey.ai/v1", + auth: { + type: "header", + apiKeyEnv: "PORTKEY_API_KEY", + headerName: "x-portkey-api-key", + prefix: "", + }, + headers: { + "x-portkey-config": { env: "PORTKEY_CONFIG_ID" }, + "x-portkey-provider": { env: "PORTKEY_PROVIDER" }, + "x-portkey-virtual-key": { env: "PORTKEY_VIRTUAL_KEY" }, + }, + enabled: true, + regions: ["global"], + dataPolicy: { + allowTraining: false, + allowLogging: true, + zeroDataRetentionAvailable: false, + byokOnly: true, + }, + }, + "cloudflare-ai-gateway": { + id: "cloudflare-ai-gateway", + displayName: "Cloudflare AI Gateway", + kind: "openai-compatible", + baseUrlEnv: "CLOUDFLARE_AI_GATEWAY_BASE_URL", + apiKeyEnv: "CLOUDFLARE_API_TOKEN", + enabled: true, + regions: ["global"], + dataPolicy: { + allowTraining: false, + allowLogging: true, + zeroDataRetentionAvailable: false, + byokOnly: true, + }, + }, + "helicone-ai-gateway": { + id: "helicone-ai-gateway", + displayName: "Helicone AI Gateway", + kind: "openai-compatible", + baseUrl: "https://ai-gateway.helicone.ai/v1", + auth: { + type: "header", + apiKeyEnv: "HELICONE_API_KEY", + headerName: "Helicone-Auth", + prefix: "Bearer ", + }, + enabled: true, + regions: ["global"], + dataPolicy: { + allowTraining: false, + allowLogging: true, + zeroDataRetentionAvailable: false, + byokOnly: true, + }, + }, + "kong-ai-gateway": { + id: "kong-ai-gateway", + displayName: "Kong AI Gateway", + kind: "openai-compatible", + baseUrlEnv: "KONG_AI_GATEWAY_BASE_URL", + apiKeyEnv: "KONG_AI_GATEWAY_API_KEY", + enabled: true, + regions: ["private"], + jurisdiction: "self-hosted", + dataPolicy: { + allowTraining: false, + allowLogging: false, + zeroDataRetentionAvailable: true, + byokOnly: true, + }, + }, deepseek: { id: "deepseek", displayName: "DeepSeek", @@ -132,6 +243,7 @@ export const providerPresets: Record = { }; export const modelPresets: GatewayModelConfig[] = [ + // --- OpenAI --- { id: "openai/gpt-4.1-mini", providerId: "openai", @@ -152,6 +264,8 @@ export const modelPresets: GatewayModelConfig[] = [ inputUsdPerMillionTokens: 0.15, outputUsdPerMillionTokens: 0.6, }, + + // --- OpenRouter --- { id: "openrouter/openai/gpt-4.1-mini", providerId: "openrouter", @@ -168,13 +282,73 @@ export const modelPresets: GatewayModelConfig[] = [ capabilities: ["chat", "streaming", "tools", "json"], contextWindow: 1_000_000, }, + { + id: "openrouter/auto", + providerId: "openrouter", + providerModel: "openrouter/auto", + aliases: ["openrouter-auto", "gateway-auto"], + capabilities: ["chat", "streaming", "tools", "json", "reasoning"], + contextWindow: 200_000, + }, + + // --- External OpenAI-compatible gateways --- + { + id: "vercel-ai-gateway/openai/gpt-4.1-mini", + providerId: "vercel-ai-gateway", + providerModel: "openai/gpt-4.1-mini", + aliases: ["vercel-fast", "vercel-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + contextWindow: 1_000_000, + }, + { + id: "litellm-proxy/coding", + providerId: "litellm-proxy", + providerModel: "coding", + aliases: ["litellm-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + }, + { + id: "portkey/openai/gpt-4.1-mini", + providerId: "portkey", + providerModel: "openai/gpt-4.1-mini", + aliases: ["portkey-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + contextWindow: 1_000_000, + }, + { + id: "cloudflare-ai-gateway/openai/gpt-4.1-mini", + providerId: "cloudflare-ai-gateway", + providerModel: "openai/gpt-4.1-mini", + aliases: ["cloudflare-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + contextWindow: 1_000_000, + }, + { + id: "helicone-ai-gateway/openai/gpt-4.1-mini", + providerId: "helicone-ai-gateway", + providerModel: "openai/gpt-4.1-mini", + aliases: ["helicone-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + contextWindow: 1_000_000, + }, + { + id: "kong-ai-gateway/coding", + providerId: "kong-ai-gateway", + providerModel: "coding", + aliases: ["kong-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + }, + + // --- DeepSeek --- { id: "deepseek/deepseek-v4-pro", providerId: "deepseek", providerModel: "deepseek-v4-pro", aliases: ["coding", "reasoning", "china-coding"], capabilities: ["chat", "streaming", "tools", "reasoning"], - contextWindow: 64_000, + contextWindow: 1_000_000, + inputUsdPerMillionTokens: 0.435, + outputUsdPerMillionTokens: 0.87, }, { id: "deepseek/deepseek-v4-flash", @@ -182,15 +356,19 @@ export const modelPresets: GatewayModelConfig[] = [ providerModel: "deepseek-v4-flash", aliases: ["fast", "cheap", "china-fast"], capabilities: ["chat", "streaming", "tools"], - contextWindow: 64_000, + contextWindow: 1_000_000, + inputUsdPerMillionTokens: 0.14, + outputUsdPerMillionTokens: 0.28, }, + + // --- Qwen / DashScope --- { id: "qwen/qwen3-coder-plus", providerId: "qwen", providerModel: "qwen3-coder-plus", aliases: ["coding", "china-coding"], - capabilities: ["chat", "streaming", "tools"], - contextWindow: 128_000, + capabilities: ["chat", "streaming", "tools", "reasoning"], + contextWindow: 1_000_000, }, { id: "qwen/qwen-plus", @@ -200,6 +378,8 @@ export const modelPresets: GatewayModelConfig[] = [ capabilities: ["chat", "streaming", "tools"], contextWindow: 128_000, }, + + // --- Kimi / Moonshot --- { id: "kimi/kimi-k2.7-code", providerId: "kimi", @@ -207,6 +387,8 @@ export const modelPresets: GatewayModelConfig[] = [ aliases: ["coding", "china-coding"], capabilities: ["chat", "streaming", "tools", "reasoning"], contextWindow: 256_000, + inputUsdPerMillionTokens: 0.95, + outputUsdPerMillionTokens: 4.0, }, { id: "kimi/kimi-k2.7-code-highspeed", @@ -215,6 +397,8 @@ export const modelPresets: GatewayModelConfig[] = [ aliases: ["fast", "china-fast"], capabilities: ["chat", "streaming", "tools", "reasoning"], contextWindow: 256_000, + inputUsdPerMillionTokens: 1.9, + outputUsdPerMillionTokens: 8.0, }, { id: "kimi/kimi-k2.6", @@ -224,19 +408,37 @@ export const modelPresets: GatewayModelConfig[] = [ capabilities: ["chat", "streaming", "tools"], contextWindow: 256_000, }, + + // --- Z.AI / GLM --- { - id: "zai/glm-5.1", + id: "zai/glm-5.2", providerId: "zai", - providerModel: "glm-5.1", + providerModel: "glm-5.2", aliases: ["reasoning", "coding", "china-reasoning", "china-coding"], capabilities: ["chat", "streaming", "tools", "reasoning"], contextWindow: 1_000_000, }, + { + id: "zai/glm-5.1", + providerId: "zai", + providerModel: "glm-5.1", + aliases: ["reasoning", "china-reasoning"], + capabilities: ["chat", "streaming", "tools", "reasoning"], + contextWindow: 200_000, + }, { id: "zai/glm-5", providerId: "zai", providerModel: "glm-5", - aliases: ["reasoning", "china-reasoning"], + aliases: ["reasoning-legacy", "china-reasoning-legacy"], + capabilities: ["chat", "streaming", "tools", "reasoning"], + contextWindow: 200_000, + }, + { + id: "zai/glm-4.7", + providerId: "zai", + providerModel: "glm-4.7", + aliases: ["reasoning-legacy"], capabilities: ["chat", "streaming", "tools", "reasoning"], contextWindow: 200_000, }, @@ -244,10 +446,12 @@ export const modelPresets: GatewayModelConfig[] = [ id: "zai/glm-4.5", providerId: "zai", providerModel: "glm-4.5", - aliases: ["reasoning-legacy", "china-reasoning-legacy"], + aliases: ["reasoning-legacy"], capabilities: ["chat", "streaming", "tools", "reasoning"], contextWindow: 128_000, }, + + // --- SiliconFlow --- { id: "siliconflow/Pro/zai-org/GLM-4.7", providerId: "siliconflow", @@ -256,6 +460,14 @@ export const modelPresets: GatewayModelConfig[] = [ capabilities: ["chat", "streaming", "reasoning"], contextWindow: 128_000, }, + { + id: "siliconflow/zai-org/GLM-5.2", + providerId: "siliconflow", + providerModel: "zai-org/GLM-5.2", + aliases: ["reasoning", "coding", "china-reasoning", "china-coding"], + capabilities: ["chat", "streaming", "tools", "reasoning"], + contextWindow: 1_000_000, + }, ]; export function isChinaProvider(provider: GatewayProviderConfig): boolean { diff --git a/src/provider-config.ts b/src/provider-config.ts new file mode 100644 index 0000000..a3336a0 --- /dev/null +++ b/src/provider-config.ts @@ -0,0 +1,77 @@ +import type { GatewayProviderConfig, GatewayProviderHeaderValue } from "./types"; + +export function providerCredentialEnv(provider: GatewayProviderConfig): string | undefined { + if (provider.auth?.type === "none") return undefined; + return provider.auth?.apiKeyEnv ?? provider.apiKeyEnv; +} + +export function providerRequiresCredential(provider: GatewayProviderConfig): boolean { + return provider.auth?.type !== "none"; +} + +export function providerBaseUrl( + provider: GatewayProviderConfig, + env: Record = process.env, +): string | undefined { + if (provider.baseUrl) return provider.baseUrl; + return provider.baseUrlEnv ? env[provider.baseUrlEnv] : undefined; +} + +function resolveHeaderValue( + name: string, + value: GatewayProviderHeaderValue, + env: Record, +): string | undefined { + if (typeof value === "string") return value; + + const rawValue = value.value ?? (value.env ? env[value.env] : undefined); + if (rawValue === undefined || rawValue.length === 0) { + if (value.required) { + throw new Error(`Provider header ${name} requires environment variable ${value.env ?? "(none)"}.`); + } + return undefined; + } + + return `${value.prefix ?? ""}${rawValue}`; +} + +export function buildProviderHeaders(input: { + provider: GatewayProviderConfig; + apiKey: string; + env?: Record; +}): Record { + const env = input.env ?? process.env; + const headers: Record = {}; + const auth = input.provider.auth ?? {}; + + if (auth.type !== "none") { + const authType = auth.type ?? "bearer"; + const headerName = auth.headerName ?? "authorization"; + const defaultPrefix = authType === "bearer" ? "Bearer " : ""; + headers[headerName] = `${auth.prefix ?? defaultPrefix}${input.apiKey}`; + } + + for (const [name, value] of Object.entries(input.provider.headers ?? {})) { + const resolved = resolveHeaderValue(name, value, env); + if (resolved !== undefined) headers[name] = resolved; + } + + return headers; +} + +export function missingRequiredProviderHeaderEnvs( + provider: GatewayProviderConfig, + env: Record = process.env, +): string[] { + const missing: string[] = []; + + for (const [name, value] of Object.entries(provider.headers ?? {})) { + if (typeof value === "string" || !value.required) continue; + const rawValue = value.value ?? (value.env ? env[value.env] : undefined); + if (rawValue === undefined || rawValue.length === 0) { + missing.push(value.env ?? name); + } + } + + return missing; +} diff --git a/src/providers/openai-compatible.ts b/src/providers/openai-compatible.ts index cbb1b38..40f25db 100644 --- a/src/providers/openai-compatible.ts +++ b/src/providers/openai-compatible.ts @@ -1,6 +1,8 @@ import { mapProviderStatus, redactSensitiveText } from "../errors"; +import { buildProviderHeaders, providerBaseUrl } from "../provider-config"; import type { GatewayModelCapability, + GatewayProviderConfig, GatewayProviderError, OpenAIChatCompletionRequest, ProviderAdapter, @@ -38,11 +40,188 @@ const forwardedFields = new Set([ "user", ]); +const openRouterProviderFields = new Set([ + "order", + "allow_fallbacks", + "require_parameters", + "data_collection", + "zdr", + "enforce_distillable_text", + "only", + "ignore", + "quantizations", + "sort", + "preferred_min_throughput", + "max_price", +]); + +const vercelGatewayFields = new Set(["models", "order", "only", "caching", "providerTimeouts"]); + function joinUrl(baseUrl: string, path: string): string { return `${baseUrl.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`; } -export function toProviderChatBody(request: OpenAIChatCompletionRequest, providerModel: string): Record { +function isObject(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + const result = value.filter((item): item is string => typeof item === "string" && item.length > 0); + return result.length ? result : undefined; +} + +function namespacedOptions(request: OpenAIChatCompletionRequest, namespace: string): Record { + const snake = isObject(request.provider_options?.[namespace]) ? request.provider_options[namespace] : undefined; + const camel = isObject(request.providerOptions?.[namespace]) ? request.providerOptions[namespace] : undefined; + return { + ...(isObject(snake) ? snake : {}), + ...(isObject(camel) ? camel : {}), + }; +} + +function pickAllowed(input: unknown, allowed: Set): Record { + if (!isObject(input)) return {}; + const output: Record = {}; + for (const [key, value] of Object.entries(input)) { + if (allowed.has(key) && value !== undefined) output[key] = value; + } + return output; +} + +function isOpenRouterProvider(provider: GatewayProviderConfig | undefined): boolean { + return provider?.id === "openrouter" || provider?.kind === "openrouter"; +} + +function isVercelGatewayProvider(provider: GatewayProviderConfig | undefined): boolean { + return provider?.id === "vercel-ai-gateway"; +} + +function openRouterProviderOptions(request: OpenAIChatCompletionRequest): Record { + const options = namespacedOptions(request, "openrouter"); + const providerOptions = { + ...pickAllowed(request.provider, openRouterProviderFields), + ...pickAllowed(options.provider, openRouterProviderFields), + }; + + if (request.gateway?.provider_order && providerOptions.order === undefined) { + providerOptions.order = request.gateway.provider_order; + } + if (request.gateway?.provider_only && providerOptions.only === undefined) { + providerOptions.only = request.gateway.provider_only; + } + if (request.gateway?.provider_ignore && providerOptions.ignore === undefined) { + providerOptions.ignore = request.gateway.provider_ignore; + } + if (request.gateway?.provider_sort && providerOptions.sort === undefined) { + providerOptions.sort = request.gateway.provider_sort; + } + if (request.gateway?.allow_fallbacks !== undefined && providerOptions.allow_fallbacks === undefined) { + providerOptions.allow_fallbacks = request.gateway.allow_fallbacks; + } + if (request.gateway?.zdr === true) { + providerOptions.zdr = true; + } else if (request.gateway?.zdr !== undefined && providerOptions.zdr === undefined) { + providerOptions.zdr = request.gateway.zdr; + } + if (request.gateway?.zero_data_retention_required) { + providerOptions.zdr = true; + } + if (request.gateway?.data_collection && providerOptions.data_collection === undefined) { + providerOptions.data_collection = request.gateway.data_collection; + } + if (request.gateway?.allow_logging === false) { + providerOptions.data_collection = "deny"; + } + if (request.gateway?.max_price && providerOptions.max_price === undefined) { + providerOptions.max_price = request.gateway.max_price; + } + + return providerOptions; +} + +function sanitizeAutoRouterPlugin(input: unknown): Record | undefined { + if (!isObject(input)) return undefined; + const plugin: Record = { id: "auto-router" }; + const allowedModels = stringArray(input.allowed_models); + if (allowedModels) plugin.allowed_models = allowedModels; + if (typeof input.cost_quality_tradeoff === "number") { + plugin.cost_quality_tradeoff = input.cost_quality_tradeoff; + } + return Object.keys(plugin).length > 1 ? plugin : undefined; +} + +function openRouterPlugins(request: OpenAIChatCompletionRequest): Record[] { + const options = namespacedOptions(request, "openrouter"); + const plugins: Record[] = []; + + for (const plugin of Array.isArray(request.plugins) ? request.plugins : []) { + const sanitized = isObject(plugin) && plugin.id === "auto-router" ? sanitizeAutoRouterPlugin(plugin) : undefined; + if (sanitized) plugins.push(sanitized); + } + + for (const plugin of Array.isArray(options.plugins) ? options.plugins : []) { + const sanitized = isObject(plugin) && plugin.id === "auto-router" ? sanitizeAutoRouterPlugin(plugin) : undefined; + if (sanitized) plugins.push(sanitized); + } + + const autoRouterOptions = { + ...pickAllowed(options, new Set(["allowed_models", "cost_quality_tradeoff"])), + ...pickAllowed(options.auto, new Set(["allowed_models", "cost_quality_tradeoff"])), + ...(request.gateway?.cost_quality_tradeoff === undefined + ? {} + : { cost_quality_tradeoff: request.gateway.cost_quality_tradeoff }), + }; + const autoRouterPlugin = sanitizeAutoRouterPlugin(autoRouterOptions); + if (autoRouterPlugin) plugins.push(autoRouterPlugin); + + return plugins; +} + +function applyOpenRouterOptions(body: Record, request: OpenAIChatCompletionRequest): void { + const provider = openRouterProviderOptions(request); + if (Object.keys(provider).length > 0) body.provider = provider; + + const plugins = openRouterPlugins(request); + if (plugins.length > 0) body.plugins = plugins; + + const options = namespacedOptions(request, "openrouter"); + const sessionId = + (typeof options.session_id === "string" ? options.session_id : undefined) ?? + request.gateway?.sticky_session_id ?? + request.gateway?.session_id ?? + request.session_id; + if (sessionId) body.session_id = sessionId; +} + +function applyVercelGatewayOptions(body: Record, request: OpenAIChatCompletionRequest): void { + const options = namespacedOptions(request, "vercel"); + const gatewayInput = isObject(options.gateway) ? options.gateway : options; + const gateway = pickAllowed(gatewayInput, vercelGatewayFields); + + if (request.gateway?.provider_order && gateway.order === undefined) { + gateway.order = request.gateway.provider_order; + } + if (request.gateway?.provider_only && gateway.only === undefined) { + gateway.only = request.gateway.provider_only; + } + if (request.gateway?.caching && gateway.caching === undefined) { + gateway.caching = request.gateway.caching; + } + if (request.gateway?.provider_timeouts && gateway.providerTimeouts === undefined) { + gateway.providerTimeouts = request.gateway.provider_timeouts; + } + + if (Object.keys(gateway).length > 0) { + body.providerOptions = { gateway }; + } +} + +export function toProviderChatBody( + request: OpenAIChatCompletionRequest, + providerModel: string, + provider?: GatewayProviderConfig, +): Record { const body: Record = {}; for (const [key, value] of Object.entries(request)) { if (key === "stream_options" && !request.stream) continue; @@ -52,6 +231,13 @@ export function toProviderChatBody(request: OpenAIChatCompletionRequest, provide } body.model = providerModel; + + if (isOpenRouterProvider(provider)) { + applyOpenRouterOptions(body, request); + } else if (isVercelGatewayProvider(provider)) { + applyVercelGatewayOptions(body, request); + } + return body; } @@ -66,25 +252,24 @@ export class OpenAICompatibleAdapter implements ProviderAdapter { readonly supports: GatewayModelCapability[] = ["chat", "streaming", "tools", "json"]; buildRequest(input: ProviderBuildInput): ProviderHttpRequest { - if (!input.provider.baseUrl) { - throw new Error(`Provider ${input.provider.id} does not define a baseUrl.`); + const baseUrl = providerBaseUrl(input.provider, input.env); + if (!baseUrl) { + throw new Error(`Provider ${input.provider.id} does not define a baseUrl or resolvable baseUrlEnv.`); } - const body = toProviderChatBody(input.request, input.model.providerModel); + const body = toProviderChatBody(input.request, input.model.providerModel, input.provider); return { - url: joinUrl(input.provider.baseUrl, "/chat/completions"), + url: joinUrl(baseUrl, "/chat/completions"), init: { method: "POST", headers: { "content-type": "application/json", - authorization: `Bearer ${input.apiKey}`, - ...(input.provider.id === "openrouter" - ? { - "http-referer": "https://github.com/hasna/open-gateway", - "x-title": "Hasna Gateway", - } - : {}), + ...buildProviderHeaders({ + provider: input.provider, + apiKey: input.apiKey, + env: input.env, + }), }, body: JSON.stringify(body), signal: createAbortSignal(input.timeoutMs, input.signal), diff --git a/src/router.ts b/src/router.ts index 091728b..b4e3462 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,5 +1,11 @@ import { GatewayHttpError } from "./errors"; import { isChinaProvider } from "./presets"; +import { + missingRequiredProviderHeaderEnvs, + providerBaseUrl, + providerCredentialEnv, + providerRequiresCredential, +} from "./provider-config"; import type { GatewayConfig, GatewayModelCapability, @@ -8,6 +14,7 @@ import type { GatewayRouteCandidate, GatewayRouteDecision, GatewayRoutePolicy, + GatewayRouteScore, GatewayRuntimeOptions, OpenAIChatCompletionRequest, } from "./types"; @@ -42,6 +49,14 @@ function arrayIntersection(a: string[] | undefined, b: string[] | undefined): st return a.filter((item) => bSet.has(item)); } +function arrayDifference(a: string[] | undefined, b: string[] | undefined): string[] | undefined { + if (!a) return undefined; + if (!b?.length) return a; + const bSet = new Set(b); + const result = a.filter((item) => !bSet.has(item)); + return result.length ? result : undefined; +} + function arrayUnion(a: string[] | undefined, b: string[] | undefined): string[] | undefined { const values = [...(a ?? []), ...(b ?? [])]; return values.length ? unique(values) : undefined; @@ -98,14 +113,16 @@ function mergePolicy( : arrayUnion(configuredBlockedRegions, requestPolicy?.blocked_regions); const configuredAllowedProviders = route?.providerAllowlist ?? routePolicy.allowedProviders ?? configPolicy.allowedProviders; const configuredBlockedProviders = route?.providerBlocklist ?? routePolicy.blockedProviders ?? configPolicy.blockedProviders; + const requestAllowedProviders = requestPolicy?.provider_only ?? requestPolicy?.allowed_providers; + const requestBlockedProviders = arrayUnion(requestPolicy?.blocked_providers, requestPolicy?.provider_ignore); return { allowedProviders: allowExpansion - ? requestPolicy?.allowed_providers ?? configuredAllowedProviders - : arrayIntersection(configuredAllowedProviders, requestPolicy?.allowed_providers), + ? requestAllowedProviders ?? configuredAllowedProviders + : arrayDifference(arrayIntersection(configuredAllowedProviders, requestAllowedProviders), requestBlockedProviders), blockedProviders: allowExpansion - ? requestPolicy?.blocked_providers ?? configuredBlockedProviders - : arrayUnion(configuredBlockedProviders, requestPolicy?.blocked_providers), + ? requestBlockedProviders ?? configuredBlockedProviders + : arrayUnion(configuredBlockedProviders, requestBlockedProviders), allowedRegions, blockedRegions, allowTraining: allowExpansion @@ -284,24 +301,58 @@ function candidateSkipReason( } if (!hasAllowedRegion(provider, policy)) return "provider region is not allowed"; if (!providerHasRequiredDataPolicy(provider, policy)) return "provider data policy is not allowed"; - if (policy.byokOnly && !provider.apiKeyEnv) return "provider is not configured for BYOK env credentials"; - if (!provider.apiKeyEnv || !env[provider.apiKeyEnv]) return `provider key env ${provider.apiKeyEnv ?? "(none)"} is not set`; + if (!providerBaseUrl(provider, env)) return `provider baseUrl env ${provider.baseUrlEnv ?? "(none)"} is not set`; + const credentialEnv = providerCredentialEnv(provider); + if (policy.byokOnly && !credentialEnv) return "provider is not configured for BYOK env credentials"; + if (providerRequiresCredential(provider) && (!credentialEnv || !env[credentialEnv])) { + return `provider key env ${credentialEnv ?? "(none)"} is not set`; + } + const missingHeaderEnvs = missingRequiredProviderHeaderEnvs(provider, env); + if (missingHeaderEnvs.length > 0) { + return `provider required header env ${missingHeaderEnvs.join(", ")} is not set`; + } if (!model.capabilities.includes("chat")) return "model does not support chat"; if (request.stream && !model.capabilities.includes("streaming")) return "model does not support streaming"; if (request.tools && request.tools.length > 0 && !model.capabilities.includes("tools")) { return "model does not support tools"; } + if (request.response_format && !model.capabilities.includes("json")) return "model does not support json output"; + for (const capability of request.gateway?.required_capabilities ?? []) { + if (!model.capabilities.includes(capability)) return `model does not support required capability ${capability}`; + } + if ( + request.gateway?.min_context_tokens !== undefined && + (model.contextWindow === undefined || model.contextWindow < request.gateway.min_context_tokens) + ) { + return "model context window is below request minimum"; + } + if ( + request.gateway?.min_quality !== undefined && + (model.qualityScore === undefined || model.qualityScore < request.gateway.min_quality) + ) { + return "model quality score is below request minimum"; + } if ( policy.maxInputUsdPerMillionTokens !== undefined && - model.inputUsdPerMillionTokens !== undefined && - model.inputUsdPerMillionTokens > policy.maxInputUsdPerMillionTokens + model.inputUsdPerMillionTokens === undefined + ) { + return "model input price is not configured for policy"; + } + if ( + policy.maxInputUsdPerMillionTokens !== undefined && + model.inputUsdPerMillionTokens! > policy.maxInputUsdPerMillionTokens ) { return "model input price exceeds policy"; } if ( policy.maxOutputUsdPerMillionTokens !== undefined && - model.outputUsdPerMillionTokens !== undefined && - model.outputUsdPerMillionTokens > policy.maxOutputUsdPerMillionTokens + model.outputUsdPerMillionTokens === undefined + ) { + return "model output price is not configured for policy"; + } + if ( + policy.maxOutputUsdPerMillionTokens !== undefined && + model.outputUsdPerMillionTokens! > policy.maxOutputUsdPerMillionTokens ) { return "model output price exceeds policy"; } @@ -309,23 +360,225 @@ function candidateSkipReason( return undefined; } -function sortCandidates(candidates: GatewayRouteCandidate[], mode: GatewayRoutePolicy["mode"]): GatewayRouteCandidate[] { - if (mode !== "cheapest") return candidates; - return [...candidates].sort((a, b) => { - const aCost = - a.model.inputUsdPerMillionTokens === undefined || a.model.outputUsdPerMillionTokens === undefined - ? Number.POSITIVE_INFINITY - : a.model.inputUsdPerMillionTokens + a.model.outputUsdPerMillionTokens; - const bCost = - b.model.inputUsdPerMillionTokens === undefined || b.model.outputUsdPerMillionTokens === undefined - ? Number.POSITIVE_INFINITY - : b.model.inputUsdPerMillionTokens + b.model.outputUsdPerMillionTokens; - return aCost - bCost; +function candidateHasConfiguredPrice(candidate: GatewayRouteCandidate): boolean { + return candidate.model.inputUsdPerMillionTokens !== undefined && candidate.model.outputUsdPerMillionTokens !== undefined; +} + +function configuredTokenPrice(candidate: GatewayRouteCandidate): number { + if (!candidateHasConfiguredPrice(candidate)) return Number.POSITIVE_INFINITY; + return candidate.model.inputUsdPerMillionTokens! + candidate.model.outputUsdPerMillionTokens!; +} + +function estimateInputTokens(request: OpenAIChatCompletionRequest): number { + if (request.gateway?.expected_input_tokens !== undefined) return request.gateway.expected_input_tokens; + const chars = request.messages.reduce((sum, message) => { + if (typeof message.content === "string") return sum + message.content.length; + if (Array.isArray(message.content)) return sum + JSON.stringify(message.content).length; + return sum; + }, 0); + return Math.max(1, Math.ceil(chars / 4)); +} + +function estimateOutputTokens(request: OpenAIChatCompletionRequest): number { + const maxTokens = request.max_completion_tokens ?? request.max_tokens; + return typeof maxTokens === "number" && maxTokens > 0 ? maxTokens : 512; +} + +function estimatedRequestCost(candidate: GatewayRouteCandidate, request: OpenAIChatCompletionRequest): number | undefined { + if (!candidateHasConfiguredPrice(candidate)) return undefined; + return ( + (estimateInputTokens(request) / 1_000_000) * candidate.model.inputUsdPerMillionTokens! + + (estimateOutputTokens(request) / 1_000_000) * candidate.model.outputUsdPerMillionTokens! + ); +} + +function clamp01(value: number): number { + if (!Number.isFinite(value)) return 0; + return Math.max(0, Math.min(1, value)); +} + +function inferredQuality(candidate: GatewayRouteCandidate): number { + if (candidate.model.qualityScore !== undefined) return candidate.model.qualityScore; + let score = 0.45; + if (candidate.model.capabilities.includes("reasoning")) score += 0.15; + if (candidate.model.capabilities.includes("tools")) score += 0.1; + if (candidate.model.capabilities.includes("json")) score += 0.05; + if (candidate.model.capabilities.includes("vision")) score += 0.05; + score += Math.min(candidate.model.contextWindow ?? 0, 1_000_000) / 1_000_000 * 0.1; + return clamp01(score); +} + +function inverseNormalize(value: number | undefined, values: Array, fallback: number): number { + if (value === undefined) return fallback; + const finite = values.filter((item): item is number => item !== undefined && Number.isFinite(item)); + if (finite.length === 0) return fallback; + const min = Math.min(...finite); + const max = Math.max(...finite); + if (min === max) return 1; + return clamp01(1 - (value - min) / (max - min)); +} + +function normalize(value: number | undefined, values: Array, fallback: number): number { + if (value === undefined) return fallback; + const finite = values.filter((item): item is number => item !== undefined && Number.isFinite(item)); + if (finite.length === 0) return fallback; + const min = Math.min(...finite); + const max = Math.max(...finite); + if (min === max) return 1; + return clamp01((value - min) / (max - min)); +} + +function hashString(value: string): number { + let hash = 2166136261; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} + +function stickyTieBreaker(candidate: GatewayRouteCandidate, request: OpenAIChatCompletionRequest): number { + const sessionId = request.gateway?.sticky_session_id ?? request.gateway?.session_id ?? request.session_id; + if (!sessionId) return 0; + return hashString(`${sessionId}:${candidate.model.id}`) / 0xffffffff; +} + +function providerOrderScore(candidate: GatewayRouteCandidate, request: OpenAIChatCompletionRequest): number | undefined { + const order = request.gateway?.provider_order; + if (!order?.length) return undefined; + const index = order.indexOf(candidate.provider.id); + if (index < 0) return 0; + return 1 - index / Math.max(order.length, 1); +} + +function weightsForMode( + mode: GatewayRoutePolicy["mode"], + request: OpenAIChatCompletionRequest, +): Record<"cost" | "quality" | "latency" | "success" | "throughput" | "providerOrder", number> { + if (mode === "lowest-latency") { + return { cost: 0.1, quality: 0.1, latency: 0.55, success: 0.2, throughput: 0, providerOrder: 0.05 }; + } + if (mode === "highest-throughput") { + return { cost: 0.1, quality: 0.1, latency: 0.1, success: 0.25, throughput: 0.4, providerOrder: 0.05 }; + } + + const priority = request.gateway?.priority ?? "balanced"; + if (priority === "cost") { + return { cost: 0.55, quality: 0.15, latency: 0.1, success: 0.15, throughput: 0, providerOrder: 0.05 }; + } + if (priority === "quality") { + return { cost: 0.1, quality: 0.55, latency: 0.1, success: 0.2, throughput: 0, providerOrder: 0.05 }; + } + if (priority === "latency") { + return { cost: 0.1, quality: 0.15, latency: 0.45, success: 0.25, throughput: 0, providerOrder: 0.05 }; + } + + const tradeoff = clamp01((request.gateway?.cost_quality_tradeoff ?? 5) / 10); + return { + cost: 0.2 + tradeoff * 0.25, + quality: 0.45 - tradeoff * 0.25, + latency: 0.15, + success: 0.15, + throughput: 0, + providerOrder: 0.05, + }; +} + +function scoreCandidates( + candidates: GatewayRouteCandidate[], + mode: GatewayRoutePolicy["mode"], + request: OpenAIChatCompletionRequest, +): GatewayRouteScore[] { + const costs = candidates.map((candidate) => estimatedRequestCost(candidate, request)); + const latencies = candidates.map((candidate) => candidate.model.averageLatencyMs); + const throughputs = candidates.map((candidate) => candidate.model.throughputTokensPerSecond); + const weights = weightsForMode(mode, request); + + return candidates.map((candidate) => { + const components = { + cost: inverseNormalize(estimatedRequestCost(candidate, request), costs, 0.35), + quality: inferredQuality(candidate), + latency: inverseNormalize(candidate.model.averageLatencyMs, latencies, 0.5), + success: candidate.model.successRate ?? 0.5, + throughput: normalize(candidate.model.throughputTokensPerSecond, throughputs, 0.5), + providerOrder: providerOrderScore(candidate, request) ?? 0.5, + sticky: stickyTieBreaker(candidate, request), + }; + const score = + components.cost * weights.cost + + components.quality * weights.quality + + components.latency * weights.latency + + components.success * weights.success + + components.throughput * weights.throughput + + components.providerOrder * weights.providerOrder; + const reason = + mode === "lowest-latency" + ? "highest latency-weighted score among eligible models" + : mode === "highest-throughput" + ? "highest throughput and success weighted score among eligible models" + : "highest cost, quality, latency, and success weighted score among eligible models"; + + return { + provider: candidate.provider.id, + model: candidate.model.id, + providerModel: candidate.model.providerModel, + score, + reason, + components, + }; }); } -function candidateHasConfiguredPrice(candidate: GatewayRouteCandidate): boolean { - return candidate.model.inputUsdPerMillionTokens !== undefined && candidate.model.outputUsdPerMillionTokens !== undefined; +function scoreFor(candidate: GatewayRouteCandidate, scores: GatewayRouteScore[]): GatewayRouteScore | undefined { + return scores.find((score) => score.model === candidate.model.id && score.provider === candidate.provider.id); +} + +function originalIndexMap(candidates: GatewayRouteCandidate[]): Map { + return new Map(candidates.map((candidate, index) => [`${candidate.provider.id}:${candidate.model.id}`, index])); +} + +function sortCandidates( + candidates: GatewayRouteCandidate[], + mode: GatewayRoutePolicy["mode"], + request: OpenAIChatCompletionRequest, +): { sorted: GatewayRouteCandidate[]; scores?: GatewayRouteScore[] } { + const indexes = originalIndexMap(candidates); + const byOriginalOrder = (a: GatewayRouteCandidate, b: GatewayRouteCandidate): number => + (indexes.get(`${a.provider.id}:${a.model.id}`) ?? 0) - (indexes.get(`${b.provider.id}:${b.model.id}`) ?? 0); + + if (mode === "cheapest") { + return { + sorted: [...candidates].sort((a, b) => configuredTokenPrice(a) - configuredTokenPrice(b) || byOriginalOrder(a, b)), + }; + } + + if (mode === "fallback" || mode === "explicit") { + const order = request.gateway?.provider_order; + if (!order?.length) return { sorted: candidates }; + return { + sorted: [...candidates].sort((a, b) => { + const aIndex = order.indexOf(a.provider.id); + const bIndex = order.indexOf(b.provider.id); + const aRank = aIndex < 0 ? Number.POSITIVE_INFINITY : aIndex; + const bRank = bIndex < 0 ? Number.POSITIVE_INFINITY : bIndex; + return aRank - bRank || byOriginalOrder(a, b); + }), + }; + } + + const scores = scoreCandidates(candidates, mode, request); + return { + scores, + sorted: [...candidates].sort((a, b) => { + const aScore = scoreFor(a, scores); + const bScore = scoreFor(b, scores); + return ( + (bScore?.score ?? 0) - (aScore?.score ?? 0) || + (bScore?.components.sticky ?? 0) - (aScore?.components.sticky ?? 0) || + byOriginalOrder(a, b) + ); + }), + }; } function policyForDecision(policy: EffectivePolicy): GatewayRouteDecision["policy"] { @@ -373,7 +626,8 @@ export function resolveRoute(options: GatewayRuntimeOptions, request: OpenAIChat } } - const sorted = sortCandidates(eligible, mode); + const { sorted, scores } = sortCandidates(eligible, mode, request); + if (scores) decision.scores = scores.sort((a, b) => b.score - a.score); if (mode === "cheapest" && sorted.length > 0 && !sorted.some(candidateHasConfiguredPrice)) { decision.reason = "no eligible model has configured token price for cheapest routing"; throw new GatewayHttpError({ @@ -387,7 +641,14 @@ export function resolveRoute(options: GatewayRuntimeOptions, request: OpenAIChat if (sorted.length > 0) { decision.selected = sorted[0]?.model.id; - decision.reason = mode === "cheapest" ? "lowest configured token price among eligible models" : "first eligible model"; + decision.reason = + mode === "cheapest" + ? "lowest configured token price among eligible models" + : scores + ? (scoreFor(sorted[0]!, scores)?.reason ?? "highest score among eligible models") + : request.gateway?.provider_order?.length + ? "first eligible model after provider_order hint" + : "first eligible model"; return { candidates: sorted, decision }; } diff --git a/src/smoke.ts b/src/smoke.ts index 2c4a78c..2d5e5d6 100644 --- a/src/smoke.ts +++ b/src/smoke.ts @@ -1,6 +1,7 @@ import { createChatCompletion } from "./gateway"; import { redactSensitiveText } from "./errors"; import { isChinaProvider } from "./presets"; +import { providerCredentialEnv, providerRequiresCredential } from "./provider-config"; import { resolveRoute } from "./router"; import type { GatewayConfig, @@ -154,11 +155,12 @@ export async function runAvailableProviderSmokeChecks(input: { continue; } - if (!provider.apiKeyEnv || !env[provider.apiKeyEnv]) { + const credentialEnv = providerCredentialEnv(provider); + if (providerRequiresCredential(provider) && (!credentialEnv || !env[credentialEnv])) { results.push({ status: "skipped", provider: provider.id, - message: `Provider ${provider.id} skipped because ${provider.apiKeyEnv ?? "apiKeyEnv"} is not set.`, + message: `Provider ${provider.id} skipped because ${credentialEnv ?? "apiKeyEnv"} is not set.`, }); continue; } diff --git a/src/types.ts b/src/types.ts index 6d20025..dfaaafb 100644 --- a/src/types.ts +++ b/src/types.ts @@ -22,7 +22,8 @@ export type GatewayRoutingMode = | "cheapest" | "lowest-latency" | "highest-throughput" - | "balanced"; + | "balanced" + | "smart"; export type GatewayRuntimeMode = "local" | "production-cloud"; @@ -43,7 +44,10 @@ export type GatewayProviderConfig = { displayName: string; kind: GatewayProviderKind; baseUrl?: string; + baseUrlEnv?: string; apiKeyEnv?: string; + auth?: GatewayProviderAuthConfig; + headers?: Record; enabled?: boolean; regions?: string[]; jurisdiction?: string; @@ -62,8 +66,28 @@ export type GatewayModelConfig = { contextWindow?: number; inputUsdPerMillionTokens?: number; outputUsdPerMillionTokens?: number; + qualityScore?: number; + averageLatencyMs?: number; + successRate?: number; + throughputTokensPerSecond?: number; }; +export type GatewayProviderAuthConfig = { + type?: "bearer" | "header" | "none"; + apiKeyEnv?: string; + headerName?: string; + prefix?: string; +}; + +export type GatewayProviderHeaderValue = + | string + | { + value?: string; + env?: string; + prefix?: string; + required?: boolean; + }; + export type GatewayRoutePolicy = { id: string; mode: GatewayRoutingMode; @@ -212,6 +236,25 @@ export type ChatMessage = { export type GatewayRequestOptions = { routing?: GatewayRoutingMode; + task?: string; + priority?: "cost" | "quality" | "latency" | "balanced"; + cost_quality_tradeoff?: number; + sticky_session_id?: string; + session_id?: string; + min_quality?: number; + min_context_tokens?: number; + expected_input_tokens?: number; + required_capabilities?: GatewayModelCapability[]; + provider_order?: string[]; + provider_only?: string[]; + provider_ignore?: string[]; + provider_sort?: string | Record; + allow_fallbacks?: boolean; + zdr?: boolean; + data_collection?: "allow" | "deny"; + max_price?: Record; + caching?: "auto"; + provider_timeouts?: Record; allowed_providers?: string[]; blocked_providers?: string[]; allowed_regions?: string[]; @@ -258,6 +301,10 @@ export type OpenAIChatCompletionRequest = { user?: string; gateway?: GatewayRequestOptions; provider_options?: Record; + providerOptions?: Record; + provider?: unknown; + plugins?: unknown; + session_id?: string; [key: string]: unknown; }; @@ -297,6 +344,15 @@ export type GatewayRouteAttempt = { latencyMs?: number; }; +export type GatewayRouteScore = { + provider: string; + model: string; + providerModel: string; + score: number; + reason: string; + components: Record; +}; + export type GatewayRouteDecision = { requested_model: string; resolved_candidates: string[]; @@ -315,6 +371,7 @@ export type GatewayRouteDecision = { }; reason: string; attempts: GatewayRouteAttempt[]; + scores?: GatewayRouteScore[]; }; export type GatewayRouteCandidate = { @@ -345,6 +402,7 @@ export type ProviderBuildInput = { request: OpenAIChatCompletionRequest; apiKey: string; timeoutMs: number; + env?: Record; fetchImpl?: GatewayFetch; signal?: AbortSignal; }; diff --git a/tests/config.test.ts b/tests/config.test.ts index ec7839f..1da9f19 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -153,6 +153,62 @@ describe("config validation", () => { } }); + test("accepts auth-header provider credentials in production-cloud mode", () => { + const config = testConfig(); + config.runtime = { + mode: "production-cloud", + serviceDiscovery: { + allowLocalProviderEndpoints: false, + allowedProviderBaseUrls: ["https://api.portkey.test"], + }, + health: { + requireRuntimeSecrets: true, + }, + }; + config.server.host = "0.0.0.0"; + config.providers = [ + { + id: "portkey", + displayName: "Portkey AI Gateway", + kind: "openai-compatible", + baseUrl: "https://api.portkey.test/v1", + auth: { + type: "header", + apiKeyEnv: "PORTKEY_API_KEY", + headerName: "x-portkey-api-key", + prefix: "", + }, + enabled: true, + regions: ["global"], + dataPolicy: { allowTraining: false, allowLogging: true, byokOnly: true }, + }, + ]; + config.models = [ + { + id: "portkey/gpt", + providerId: "portkey", + providerModel: "openai/gpt-4.1-mini", + capabilities: ["chat"], + }, + ]; + config.routes = [ + { + id: "portkey-coding", + mode: "fallback", + modelAliases: ["portkey-coding"], + fallbackModelIds: ["portkey/gpt"], + dataPolicy: { allowTraining: false, allowLogging: true, allowedRegions: ["global"] }, + }, + ]; + + const result = validateConfig(config); + expect(result.ok).toBe(true); + if (result.ok) { + // Credential lives in auth.apiKeyEnv, not top-level apiKeyEnv; must not be rejected. + expect(result.config.providers[0]?.auth?.apiKeyEnv).toBe("PORTKEY_API_KEY"); + } + }); + test("rejects models with unknown providers", () => { const config = testConfig(); config.models[0] = { @@ -529,4 +585,90 @@ describe("config validation", () => { validateRuntimeSecrets(config, { GATEWAY_API_KEY: "gateway", OPENAI_API_KEY: "openai" }), ).toEqual([]); }); + + test("accepts baseUrlEnv, custom auth, and env-derived provider headers", () => { + const result = validateConfig({ + providers: [ + { + id: "portkey", + displayName: "Portkey", + kind: "openai-compatible", + baseUrlEnv: "PORTKEY_BASE_URL", + auth: { + type: "header", + apiKeyEnv: "PORTKEY_API_KEY", + headerName: "x-portkey-api-key", + prefix: "", + }, + headers: { + "x-portkey-config": { env: "PORTKEY_CONFIG_ID" }, + }, + dataPolicy: { allowTraining: false, allowLogging: true, byokOnly: true }, + }, + ], + models: [ + { + id: "portkey/test", + providerId: "portkey", + providerModel: "openai/gpt-4.1-mini", + capabilities: ["chat"], + qualityScore: 0.8, + averageLatencyMs: 1000, + successRate: 0.99, + throughputTokensPerSecond: 80, + }, + ], + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.config.providers[0]?.baseUrlEnv).toBe("PORTKEY_BASE_URL"); + expect(result.config.providers[0]?.auth?.apiKeyEnv).toBe("PORTKEY_API_KEY"); + expect(result.config.models[0]?.qualityScore).toBe(0.8); + } + }); + + test("runtime secret validation uses auth.apiKeyEnv", () => { + const result = validateConfig({ + auth: { + apiKeyEnv: "GATEWAY_API_KEY", + required: true, + }, + providers: [ + { + id: "custom", + displayName: "Custom", + kind: "openai-compatible", + baseUrl: "https://custom.test/v1", + auth: { + type: "header", + apiKeyEnv: "CUSTOM_PROVIDER_KEY", + headerName: "x-api-key", + }, + dataPolicy: { allowTraining: false, byokOnly: true }, + }, + ], + models: [ + { + id: "custom/test", + providerId: "custom", + providerModel: "test", + capabilities: ["chat"], + }, + ], + }); + + expect(result.ok).toBe(true); + if (result.ok) { + expect(validateRuntimeSecrets(result.config, { GATEWAY_API_KEY: "gateway" })).toContain( + "At least one enabled provider must have its apiKeyEnv set in the environment.", + ); + expect( + validateRuntimeSecrets(result.config, { + GATEWAY_API_KEY: "gateway", + CUSTOM_PROVIDER_KEY: "provider", + }), + ).toEqual([]); + } + }); }); diff --git a/tests/gateway.test.ts b/tests/gateway.test.ts index b1a0556..2f75e77 100644 --- a/tests/gateway.test.ts +++ b/tests/gateway.test.ts @@ -785,4 +785,78 @@ describe("chat completion lifecycle", () => { ), ).rejects.toBeInstanceOf(GatewayHttpError); }); + + test("forwards effective route policy to OpenRouter option mapping", async () => { + const config = testConfig(); + config.providers.push({ + id: "openrouter", + displayName: "OpenRouter", + kind: "openai-compatible", + baseUrl: "https://openrouter.test/api/v1", + apiKeyEnv: "OPENROUTER_API_KEY", + enabled: true, + regions: ["global"], + dataPolicy: { + allowTraining: false, + allowLogging: false, + byokOnly: true, + zeroDataRetentionAvailable: true, + }, + }); + config.models.push({ + id: "openrouter/auto", + providerId: "openrouter", + providerModel: "openrouter/auto", + aliases: ["or-auto"], + capabilities: ["chat", "streaming", "tools", "json"], + }); + config.routes.push({ + id: "or-auto", + mode: "fallback", + modelAliases: ["or-auto"], + fallbackModelIds: ["openrouter/auto"], + dataPolicy: { + allowTraining: false, + allowLogging: false, + zeroDataRetentionRequired: true, + allowedRegions: ["global"], + }, + }); + + const fetchImpl = async (_url: string | URL | Request, init?: RequestInit): Promise => { + const body = JSON.parse(String(init?.body)); + expect(body.provider).toEqual({ + zdr: true, + data_collection: "deny", + }); + return jsonResponse({ + id: "provider-id", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + }; + + await createChatCompletion( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENROUTER_API_KEY: "openrouter", + }, + fetchImpl, + }, + { + model: "or-auto", + messages: [{ role: "user", content: "hi" }], + provider_options: { + openrouter: { + provider: { + zdr: false, + data_collection: "allow", + }, + }, + }, + }, + ); + }); }); diff --git a/tests/provider.test.ts b/tests/provider.test.ts index 20b3b2a..51ed779 100644 --- a/tests/provider.test.ts +++ b/tests/provider.test.ts @@ -71,6 +71,153 @@ describe("OpenAI-compatible provider adapter", () => { expect((request.init.headers as Record).authorization).toBe("Bearer secret"); expect(JSON.parse(String(request.init.body)).model).toBe("gpt-4.1-mini"); }); + + test("builds custom auth and env-derived headers", () => { + const config = testConfig(); + const provider = { + ...config.providers[0]!, + id: "portkey", + baseUrl: undefined, + baseUrlEnv: "PORTKEY_BASE_URL", + apiKeyEnv: undefined, + auth: { + type: "header" as const, + apiKeyEnv: "PORTKEY_API_KEY", + headerName: "x-portkey-api-key", + prefix: "", + }, + headers: { + "x-portkey-config": { env: "PORTKEY_CONFIG_ID" }, + "x-static": "static-value", + }, + }; + const model = { ...config.models[0]!, providerId: "portkey" }; + const adapter = new OpenAICompatibleAdapter(); + + const request = adapter.buildRequest({ + provider, + model, + request: { + model: "portkey-coding", + messages: [{ role: "user", content: "hi" }], + }, + apiKey: "pk-key", + timeoutMs: 1000, + env: { + PORTKEY_BASE_URL: "https://portkey.test/v1", + PORTKEY_CONFIG_ID: "pc-test", + }, + }); + + expect(request.url).toBe("https://portkey.test/v1/chat/completions"); + expect((request.init.headers as Record)["x-portkey-api-key"]).toBe("pk-key"); + expect((request.init.headers as Record)["x-portkey-config"]).toBe("pc-test"); + expect((request.init.headers as Record)["x-static"]).toBe("static-value"); + expect((request.init.headers as Record).authorization).toBeUndefined(); + }); + + test("maps only supported OpenRouter provider and auto-router options", () => { + const config = testConfig(); + const provider = { + ...config.providers[0]!, + id: "openrouter", + kind: "openai-compatible" as const, + }; + const body = toProviderChatBody( + { + model: "openrouter-auto", + messages: [{ role: "user", content: "hi" }], + gateway: { + provider_order: ["anthropic", "openai"], + provider_only: ["anthropic", "openai"], + provider_ignore: ["bad-provider"], + provider_sort: "latency", + allow_fallbacks: false, + zero_data_retention_required: true, + allow_logging: false, + max_price: { prompt: 1, completion: 2 }, + cost_quality_tradeoff: 3, + sticky_session_id: "session-1", + }, + provider_options: { + openrouter: { + allowed_models: ["anthropic/*"], + provider: { + ignore: ["deepinfra"], + zdr: false, + data_collection: "allow", + unsupported_secret: "drop-me", + }, + apiKey: "drop-me-too", + }, + }, + }, + "openrouter/auto", + provider, + ); + + expect(body.model).toBe("openrouter/auto"); + expect(body.provider).toEqual({ + order: ["anthropic", "openai"], + only: ["anthropic", "openai"], + ignore: ["deepinfra"], + sort: "latency", + allow_fallbacks: false, + zdr: true, + data_collection: "deny", + max_price: { prompt: 1, completion: 2 }, + }); + expect(body.plugins).toEqual([ + { + id: "auto-router", + allowed_models: ["anthropic/*"], + cost_quality_tradeoff: 3, + }, + ]); + expect(body.session_id).toBe("session-1"); + expect(body.provider_options).toBeUndefined(); + }); + + test("maps only Vercel AI Gateway provider options", () => { + const config = testConfig(); + const provider = { + ...config.providers[0]!, + id: "vercel-ai-gateway", + }; + const body = toProviderChatBody( + { + model: "vercel-coding", + messages: [{ role: "user", content: "hi" }], + gateway: { + provider_order: ["bedrock", "anthropic"], + provider_only: ["bedrock", "anthropic"], + caching: "auto", + provider_timeouts: { byok: { anthropic: 3000 } }, + }, + providerOptions: { + vercel: { + gateway: { + only: ["anthropic"], + byok: { openai: "drop-secret" }, + }, + }, + }, + }, + "openai/gpt-4.1-mini", + provider, + ); + + expect(body.providerOptions).toEqual({ + gateway: { + only: ["anthropic"], + order: ["bedrock", "anthropic"], + caching: "auto", + providerTimeouts: { byok: { anthropic: 3000 } }, + }, + }); + expect(body.provider_options).toBeUndefined(); + expect((body.providerOptions as Record>).gateway.byok).toBeUndefined(); + }); }); describe("Anthropic Messages provider adapter", () => { diff --git a/tests/router.test.ts b/tests/router.test.ts index df0f54f..8826724 100644 --- a/tests/router.test.ts +++ b/tests/router.test.ts @@ -378,6 +378,13 @@ describe("routing policy", () => { test("skips models above configured price policy", () => { const config = testConfig(); + // Give the fallback a configured price under the ceiling so it stays eligible; + // price ceilings are fail-closed for unpriced models under smart-routing policy. + config.models = config.models.map((model) => + model.id === "deepseek/deepseek-v4-pro" + ? { ...model, inputUsdPerMillionTokens: 0.05, outputUsdPerMillionTokens: 0.1 } + : model, + ); config.routes[0] = { ...config.routes[0]!, maxInputUsdPerMillionTokens: 0.1, @@ -397,4 +404,268 @@ describe("routing policy", () => { ); expect(result.decision.selected).toBe("deepseek/deepseek-v4-pro"); }); + + test("fails closed when price ceilings require an unpriced model", () => { + const config = testConfig(); + config.routes.push({ + id: "unpriced", + mode: "fallback", + modelAliases: ["unpriced"], + fallbackModelIds: ["deepseek/deepseek-v4-pro"], + dataPolicy: { + allowTraining: false, + allowLogging: true, + allowChineseProviders: true, + allowedRegions: ["cn"], + }, + maxInputUsdPerMillionTokens: 0, + maxOutputUsdPerMillionTokens: 0, + }); + + expect(() => + resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + DEEPSEEK_API_KEY: "deepseek", + }, + }, + { + ...request, + model: "unpriced", + }, + ), + ).toThrow(GatewayHttpError); + }); + + test("fails closed when required provider header env is missing", () => { + const config = testConfig(); + config.providers[0] = { + ...config.providers[0]!, + headers: { + "x-required-config": { + env: "REQUIRED_PROVIDER_CONFIG", + required: true, + }, + }, + }; + + expect(() => + resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + }, + }, + { + ...request, + model: "openai/gpt-4.1-mini", + }, + ), + ).toThrow(GatewayHttpError); + }); + + test("smart routing scores eligible candidates by request priority", () => { + const config = testConfig(); + config.models = [ + ...config.models, + { + id: "openai/cheap", + providerId: "openai", + providerModel: "cheap", + aliases: ["smart-coding"], + capabilities: ["chat", "streaming", "tools", "json"], + contextWindow: 128_000, + inputUsdPerMillionTokens: 0.1, + outputUsdPerMillionTokens: 0.2, + qualityScore: 0.45, + averageLatencyMs: 500, + successRate: 0.98, + }, + { + id: "openai/quality", + providerId: "openai", + providerModel: "quality", + aliases: ["smart-coding"], + capabilities: ["chat", "streaming", "tools", "json", "reasoning"], + contextWindow: 1_000_000, + inputUsdPerMillionTokens: 5, + outputUsdPerMillionTokens: 10, + qualityScore: 0.95, + averageLatencyMs: 1200, + successRate: 0.99, + }, + ]; + config.routes.push({ + id: "smart-coding", + mode: "smart", + modelAliases: ["smart-coding"], + fallbackModelIds: ["openai/cheap", "openai/quality"], + dataPolicy: { allowTraining: false, allowLogging: false, blockedRegions: ["cn"] }, + }); + + const qualityResult = resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + }, + }, + { + ...request, + model: "smart-coding", + gateway: { priority: "quality" }, + }, + ); + expect(qualityResult.decision.selected).toBe("openai/quality"); + expect(qualityResult.decision.scores?.[0]?.model).toBe("openai/quality"); + + const costResult = resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + }, + }, + { + ...request, + model: "smart-coding", + gateway: { priority: "cost" }, + }, + ); + expect(costResult.decision.selected).toBe("openai/cheap"); + }); + + test("policy filtering happens before smart scoring", () => { + const config = testConfig(); + config.models = [ + ...config.models, + { + id: "openai/safe", + providerId: "openai", + providerModel: "safe", + aliases: ["policy-smart"], + capabilities: ["chat", "streaming", "tools"], + qualityScore: 0.4, + inputUsdPerMillionTokens: 0.4, + outputUsdPerMillionTokens: 1, + }, + { + id: "deepseek/high-quality", + providerId: "deepseek", + providerModel: "high-quality", + aliases: ["policy-smart"], + capabilities: ["chat", "streaming", "tools", "reasoning"], + qualityScore: 1, + inputUsdPerMillionTokens: 0.1, + outputUsdPerMillionTokens: 0.2, + }, + ]; + config.routes.push({ + id: "policy-smart", + mode: "smart", + modelAliases: ["policy-smart"], + fallbackModelIds: ["deepseek/high-quality", "openai/safe"], + dataPolicy: { + allowTraining: false, + allowLogging: false, + allowChineseProviders: false, + blockedRegions: ["cn"], + }, + }); + + const result = resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + DEEPSEEK_API_KEY: "deepseek", + }, + }, + { + ...request, + model: "policy-smart", + gateway: { priority: "quality" }, + }, + ); + + expect(result.decision.selected).toBe("openai/safe"); + expect(result.decision.scores?.map((score) => score.model)).toEqual(["openai/safe"]); + expect(result.decision.attempts[0]?.status).toBe("skipped"); + expect(result.decision.attempts[0]?.reason).toContain("china provider"); + }); + + test("required capabilities and context fail closed", () => { + expect(() => + resolveRoute( + { + config: testConfig(), + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + }, + }, + { + ...request, + model: "openai/gpt-4.1-mini", + gateway: { + required_capabilities: ["vision"], + min_context_tokens: 2_000_000, + }, + }, + ), + ).toThrow(GatewayHttpError); + }); + + test("unknown smart metrics fall back to deterministic configured order", () => { + const config = testConfig(); + config.models = [ + { + id: "openai/first", + providerId: "openai", + providerModel: "first", + aliases: ["unknown-metrics"], + capabilities: ["chat"], + }, + { + id: "openai/second", + providerId: "openai", + providerModel: "second", + aliases: ["unknown-metrics"], + capabilities: ["chat"], + }, + ]; + config.routes = [ + { + id: "unknown-metrics", + mode: "smart", + modelAliases: ["unknown-metrics"], + fallbackModelIds: ["openai/first", "openai/second"], + dataPolicy: { allowTraining: false, allowLogging: false, blockedRegions: ["cn"] }, + }, + ]; + + const result = resolveRoute( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + }, + }, + { + ...request, + model: "unknown-metrics", + }, + ); + + expect(result.decision.selected).toBe("openai/first"); + expect(result.decision.scores?.length).toBe(2); + }); });