From 1d3ac128672d660a63b6d522c82ed0e3adb1f967 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 29 Jul 2026 12:07:15 +0300 Subject: [PATCH 1/3] feat: docs deep-scan: read the whole of gateway (src + bins + README + MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs deep-scan: read the whole of gateway (src + bins + README + docs/) and update all docs to match current behavior — README accuracy, CLI help parity, missing docs pages; PR-first, conventional commits [loop-infinity-300pr] X-Factory-Run: run_dfaf22bfaddb X-Factory-Task: c361f87f-626a-4430-9b56-2ab81b282853 --- README.md | 15 +++++--- docs/README.md | 30 ++++++++-------- docs/api-contract.md | 40 +++++++++++++++++++--- docs/cli.md | 48 ++++++++++++++++++++++++++ docs/configuration.md | 78 ++++++++++++++++++++++++++++++++++++++++++ src/cli/index.ts | 21 ++++++++---- tests/cli-help.test.ts | 30 ++++++++++++++++ 7 files changed, 231 insertions(+), 31 deletions(-) create mode 100644 docs/cli.md create mode 100644 docs/configuration.md create mode 100644 tests/cli-help.test.ts diff --git a/README.md b/README.md index 9d13577..f2fcc7a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The open-source package is useful on its own. Anyone can run it locally or on th ## Product Shape -- OpenAI-compatible HTTP API first, starting with `/v1/chat/completions`. +- OpenAI-compatible chat completions and embeddings APIs. - One gateway key for clients, many provider keys behind the gateway. - Bring-your-own-key mode for self-hosted users. - Routing by model alias, provider allowlist/blocklist, region policy, price ceilings, fallback, capability, and smart cost/quality/latency hints. @@ -72,8 +72,9 @@ gateway-mcp --config gateway.config.json ``` `gateway-serve` is the package-level service binary for local and self-hosted -HTTP runtime smoke checks. It exposes `GET /health`, authenticated `GET /ready`, -`GET /version`, `GET /v1/models`, and `POST /v1/chat/completions`. +HTTP runtime smoke checks. It exposes public `GET /health` and `GET /version`, +plus `GET /ready`, `GET /v1/models`, `POST /v1/chat/completions`, and +`POST /v1/embeddings`. The latter endpoints enforce gateway auth when required. `gateway-mcp` is a stdio MCP server for local agents. It validates and inspects config, explains route choices without provider calls, manages budget definitions, checks remaining budgets, and summarizes the configured usage ledger. Long-running `serve` and live `smoke` checks stay CLI-only. See [Gateway MCP server](docs/mcp.md). @@ -86,7 +87,7 @@ Required config examples: - `gateway.config.no-china.example.json`: OpenAI/OpenRouter-only policy with `cn` blocked. - `gateway.config.china.example.json`: Chinese provider routes with explicit `cn`/`sg` allowance. -Provider keys are loaded from environment variables only. Do not put provider secrets in config files. +Built-in provider credentials are loaded from environment variables. Custom static headers can contain arbitrary values, but provider secrets should not be stored in config files. Providers can use `baseUrl`, `baseUrlEnv`, `apiKeyEnv`, custom `auth`, and static or env-derived `headers`. This keeps OpenAI-compatible gateways on the generic adapter instead of adding hardcoded adapter forks. The built-in presets include: @@ -95,6 +96,8 @@ Providers can use `baseUrl`, `baseUrlEnv`, `apiKeyEnv`, custom `auth`, and stati Smart routing is available with route mode `smart` or request `gateway.routing: "smart"`. It filters by policy first, then scores eligible candidates using configured prices, context, capabilities, quality/latency/success/throughput hints, and deterministic fallback ordering when metrics are missing. +Optional non-streaming chat response caching is configured under `server.responseCache`. Cache entries are in-memory and process-local, only successful responses are cached, budget and rate-limit accounting still runs on cache hits, and clients can bypass lookup with the configured `bypassHeader`. Streaming and embeddings requests are never cached. + ```json { "model": "coding", @@ -131,6 +134,8 @@ The companion `open-router` repo is currently documented as the future extractio ## Documentation +- [CLI reference](docs/cli.md) +- [Configuration reference](docs/configuration.md) - [Product requirements](docs/product-requirements.md) - [Architecture](docs/architecture.md) - [API contract](docs/api-contract.md) @@ -147,6 +152,6 @@ The companion `open-router` repo is currently documented as the future extractio ## Status -The gateway core is implemented and locally verified for the first release surface: CLI server, MCP server, health/models/chat endpoints, OpenAI-compatible provider adapter, provider presets, routing policy, fallbacks, streaming, usage normalization, optional local ledger, examples, tests, build, and package dry-run. +The gateway core is implemented and locally verified for the current release surface: CLI and standalone HTTP servers, MCP server, health/readiness/version/models/chat/embeddings endpoints, OpenAI-compatible and Anthropic adapters, provider presets, routing policy, fallbacks, streaming, response caching, rate limits, budgets, usage normalization, local and cloud ledgers, examples, tests, build, and package dry-run. Publication is gated on a passing live smoke check with valid provider credentials. diff --git a/docs/README.md b/docs/README.md index 6a5ef00..4dd5c0e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,20 +1,19 @@ # Hasna Gateway Docs -Read these documents before implementation: +Use these pages as the current operator and API reference: -1. [Product requirements](product-requirements.md) -2. [Architecture](architecture.md) +1. [CLI reference](cli.md) +2. [Configuration reference](configuration.md) 3. [API contract](api-contract.md) -4. [Provider adapters](provider-adapters.md) -5. [2026 provider references](provider-references.md) -6. [Routing and policy](routing-and-policy.md) -7. [Gateway MCP server](mcp.md) -8. [Open-core boundary](open-core-boundary.md) +4. [Architecture](architecture.md) +5. [Provider adapters](provider-adapters.md) +6. [2026 provider references](provider-references.md) +7. [Routing and policy](routing-and-policy.md) +8. [Gateway MCP server](mcp.md) 9. [Security and compliance](security-compliance.md) -10. [Implementation plan](implementation-plan.md) -11. [Publishing and release](publishing-and-release.md) -12. [Hasna app migration plan](migration-plan.md) -13. [Codewith handoff prompt](handoff-prompt.md) +10. [Open-core boundary](open-core-boundary.md) + +The [product requirements](product-requirements.md), [implementation plan](implementation-plan.md), [publishing and release checklist](publishing-and-release.md), [Hasna app migration plan](migration-plan.md), and [agent handoff prompt](handoff-prompt.md) record product intent and project history. When they describe future work, the current reference pages and implementation take precedence. ## Current Decision @@ -22,12 +21,13 @@ The gateway should be open source as a self-hostable core. The commercial Hasna ## Build Contract -The first implementation should prioritize a small working gateway over broad incomplete abstractions: +The implemented gateway surface includes: - A working CLI server. - A stdio MCP server for safe local gateway inspection, route explanation, budget maintenance, and ledger summaries. - OpenAI-compatible chat completions. -- OpenAI-compatible provider adapter. +- OpenAI-compatible embeddings. +- OpenAI-compatible and Anthropic provider adapters. - Config validation. - Model aliases. - Fallback routing. @@ -35,6 +35,8 @@ The first implementation should prioritize a small working gateway over broad in - Explicit provider policy. - Config-driven provider auth and headers. - Streaming. +- Optional in-memory response caching and per-key rate limits. +- Hard and soft budgets backed by local JSONL, SQLite, or Postgres usage storage. - Usage normalization. - Tests. diff --git a/docs/api-contract.md b/docs/api-contract.md index 11d9457..268a97a 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -2,7 +2,7 @@ ## Compatibility Principle -The public API should be OpenAI-compatible where possible. Existing OpenAI SDK clients should be able to point `baseURL` at the gateway and keep using `chat.completions.create`. +The public API is OpenAI-compatible where possible. Existing OpenAI SDK clients can point `baseURL` at the gateway and use chat completions and embeddings. Compatibility does not mean hiding provider differences. If a provider cannot support a feature, the gateway should return a clear capability error or route to an allowed provider that can support it. @@ -20,7 +20,7 @@ Hosted Hasna mode: Authorization: Bearer ``` -Provider keys are never sent by the client unless explicit BYOK request support is added. Initial BYOK should be configured through server-side environment variables. +Built-in provider credentials are configured through server-side environment variables. Custom provider headers may be static or environment-derived, but client request bodies never supply provider credentials. ## `GET /health` @@ -29,7 +29,7 @@ Returns service status. In `local` runtime mode this is a lightweight liveness c ```json { "status": "ok", - "version": "0.1.0", + "version": "0.1.6", "runtime": { "mode": "production-cloud" }, @@ -39,7 +39,15 @@ Returns service status. In `local` runtime mode this is a lightweight liveness c } ``` -Readiness responses must not include secret values or environment variable names. +Health responses must not include secret values or environment variable names. + +## `GET /version` + +Returns `{ "name": "@hasna/gateway", "version": "" }`. This endpoint is public. + +## `GET /ready` + +Returns authenticated operational checks for runtime config, gateway auth, providers, routes, and usage-ledger availability. The response status is `200` when ready and `503` when runtime validation fails. A missing cumulative ledger is reported as `deferred` because per-request budgets remain usable. ## `GET /v1/models` @@ -62,7 +70,7 @@ Returns configured gateway models and aliases, including provider and capability ## `POST /v1/chat/completions` -The initial critical endpoint. It should support: +Supports: - `model` - `messages` @@ -73,8 +81,12 @@ The initial critical endpoint. It should support: - `temperature` - `top_p` - `max_tokens` +- `max_completion_tokens` - `stop` - `seed` when provider supports it +- `n`, `presence_penalty`, and `frequency_penalty` +- `parallel_tool_calls`, `logprobs`, and `top_logprobs` +- `metadata`, `store`, `reasoning_effort`, `modalities`, `audio`, `prediction`, `service_tier`, and `user` Example: @@ -110,6 +122,24 @@ Unsupported gateway-only fields and secrets are stripped. Smart routing fields include `task`, `priority`, `cost_quality_tradeoff`, `sticky_session_id`, `min_quality`, `min_context_tokens`, `expected_input_tokens`, `required_capabilities`, `provider_order`, `provider_only`, and `provider_ignore`. Policy is applied before scoring. +Successful non-streaming chat responses can use the optional process-local response cache. Streaming responses, embeddings, and errors are not cached. Sending a truthy value in the configured cache bypass header skips lookup. + +## `POST /v1/embeddings` + +Accepts OpenAI-compatible `model` and `input` fields. `input` may be a string, string array, token array, or array of token arrays. Optional forwarded fields are `encoding_format`, `dimensions`, and `user`. The gateway-only `gateway` field may narrow route policy but is stripped before provider forwarding. + +Embedding route candidates must declare the `embeddings` capability. The gateway applies the same auth, policy, budget, fallback, rate-limit, usage-ledger, and metadata rules as non-streaming chat requests. Providers without an embeddings adapter are skipped. Streaming and response caching do not apply. + +```json +{ + "model": "embeddings", + "input": ["first document", "second document"], + "encoding_format": "float" +} +``` + +The response preserves the provider's OpenAI-compatible `data` array, rewrites `model` to the configured gateway model id, normalizes usage to `prompt_tokens` and `total_tokens`, and includes `gateway` metadata when enabled. + ## Response Shape Non-streaming responses should match OpenAI chat completion shape: diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..bf13963 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,48 @@ +# CLI Reference + +The package installs three binaries: + +- `gateway`: configuration, routing, budget, smoke, and server commands. +- `gateway-serve`: a dedicated HTTP server entrypoint. +- `gateway-mcp`: the stdio MCP server documented in [Gateway MCP server](mcp.md). + +All config-aware `gateway` commands default to `gateway.config.json`. Use `--config ` to select another file. + +## General + +| Command | Behavior | +| --- | --- | +| `gateway --help` or `gateway help` | Print command usage. `--help` after a command also prints the top-level usage. | +| `gateway --version` | Print the package version. | +| `gateway validate [--config ]` | Validate raw JSON config, print warnings, and exit non-zero on errors. | +| `gateway serve [--config ] [--host ] [--port ]` | Load config, validate runtime secrets, and start the HTTP server. CLI host and port override config. | +| `gateway smoke [--config ] [--model ]` | Send one live chat smoke request. The model defaults to `fast`; missing credentials produce a skipped result. | +| `gateway smoke [--config ] --all` | Smoke-test every available provider and exit non-zero if any check fails or none pass. | + +`gateway-serve` accepts `--config`, `--host`, `--port`, `--help`/`-h`, and `--version`/`-v`. Unlike `gateway serve`, invalid `--port` text is passed to `Bun.serve` as `NaN` rather than falling back to the configured port. + +## Routing + +| Command | Options and output | +| --- | --- | +| `gateway route --model ` | Dry-run chat route selection without provider traffic. Add `--stream` to require streaming capability, `--json` for the raw decision, or `--json --contract` for `hasna.decision_envelope.v1`. | +| `gateway routes` | List configured route ids. Add `--json` for route summaries or `--json --contract` for `hasna.capability_card.v1` records. | + +Route failures exit non-zero. JSON modes include the rejected route decision when one is available. + +## Budgets + +| Command | Options and output | +| --- | --- | +| `gateway budget-add --id ` | Add or replace a config budget. `--window` defaults to `lifetime`; `--mode` defaults to `hard`. Scope with `--gateway-key`, `--tenant`, and `--model`. Limits are `--max-usd`, `--max-input-tokens`, `--max-output-tokens`, `--max-total-tokens`, and `--warning-threshold`. Add `--json` for structured output. | +| `gateway budget-list` | List budget ids, or full normalized definitions with `--json`. | +| `gateway budget-remaining` | Calculate matching statuses. Filter with `--id`, `--tenant`, and `--model`; add `--json`, or `--json --contract` for `hasna.cost_estimate.v1`. | +| `gateway budget-reset --id ` | Set the budget's `resetAt` to the current time. Add `--json` for structured output. | + +Numeric budget flags must be non-negative. `budget-add` validates the complete config before writing it. + +## Local Removal + +`gateway uninstall --yes` removes the selected config file and its configured local JSONL usage ledger. `gateway remove --all --yes` is an alias with an additional explicit `--all` safeguard. Both refuse to continue without a bare `--yes`, refuse to remove directories, tolerate an already-absent ledger, and support `--json` output. The selected config must exist so the command can discover the ledger path. + +These commands do not remove the npm package, environment variables, SQLite/Postgres data, or any other file referenced by the config. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..ee9c979 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,78 @@ +# Configuration Reference + +Gateway config is JSON. `loadGatewayConfig` interpolates `${ENV_VAR}` placeholders before normalization and validation; provider credentials are still read from environment variables at runtime and should not be stored directly in config. + +Start with one of the checked-in `gateway.config.*.json` examples. Run `gateway validate --config ` after changes. + +## Defaults + +Omitted top-level values normalize to these local-first defaults: + +```json +{ + "runtime": { + "mode": "local", + "serviceDiscovery": { "allowLocalProviderEndpoints": true }, + "health": { "requireRuntimeSecrets": false } + }, + "server": { + "host": "127.0.0.1", + "port": 8787, + "requestTimeoutMs": 60000, + "maxRequestBodyBytes": 1000000, + "includeGatewayMetadata": true, + "maxFallbackAttempts": 3, + "corsAllowedOrigins": ["http://127.0.0.1:8787", "http://localhost:8787"], + "responseCache": { + "enabled": false, + "ttlMs": 300000, + "maxEntries": 500, + "bypassHeader": "x-gateway-cache-bypass" + } + }, + "auth": { "apiKeyEnv": "GATEWAY_API_KEY", "required": true }, + "storage": {}, + "policy": { + "allowTraining": false, + "allowLogging": false, + "allowChineseProviders": false, + "byokOnly": true + }, + "providers": [], + "models": [], + "routes": [], + "budgets": [] +} +``` + +## Runtime And Server + +- `runtime.mode`: `local` or `production-cloud`. Production mode requires gateway auth, non-loopback binding, secret-aware health, and cloud-safe provider URLs. +- `runtime.serviceDiscovery.allowedProviderBaseUrls`: optional exact-origin allowlist for enabled provider base URLs. +- `server.rateLimits.perGatewayKey`: optional `requestsPerMinute` and `tokensPerMinute` limits. Keys are SHA-256 fingerprints of bearer tokens; unauthenticated optional-auth traffic shares an anonymous bucket. +- `server.responseCache`: process-local cache for successful, non-streaming chat completions. It does not cache streams, errors, or embeddings. TTL and maximum entries must be positive. A truthy configured bypass header skips lookup, but successful responses can still populate the cache. +- `server.corsAllowedOrigins`: exact allowed origins. CORS preflight permits `authorization`, `content-type`, `x-gateway-tenant`, and the configured cache bypass header. + +`GET /health` and `GET /version` are public. `GET /ready` and every `/v1/*` endpoint enforce gateway auth when `auth.required` is true. + +## Providers And Models + +Providers declare `id`, `displayName`, `kind`, endpoint/auth settings, regions, and data policy. `baseUrlEnv` overrides `baseUrl`. `auth` supports bearer, custom-header, and no-auth modes; `headers` supports static values and env-derived values with optional prefixes and required markers. + +Models declare a stable gateway `id`, `providerId`, upstream `providerModel`, aliases, and capabilities. Optional context, price, quality, latency, success-rate, and throughput values drive policy filters and smart scoring. The `embeddings` capability is required for embedding routes. + +Provider and model presets are appended after explicit user definitions and deduplicated by id, so an explicit definition wins over a preset with the same id. See [Provider adapters](provider-adapters.md) for preset ids and adapter behavior. + +## Routes And Request Policy + +Routes select one of `explicit`, `fallback`, `cheapest`, `lowest-latency`, `highest-throughput`, `balanced`, or `smart`. They can bind aliases, provider allow/block lists, price and latency ceilings, fallback model ids, and data policy. + +Request `gateway` policy can narrow configured policy. It can expand policy only when `policy.allowRequestPolicyExpansion` is true. See [Routing and policy](routing-and-policy.md) for filtering and scoring order. + +## Storage And Budgets + +- `storage.usageLedgerPath`: append-only local JSONL ledger. +- `storage.cloud.backend: "sqlite"`: SQLite ledger using `sqlitePath`. +- `storage.cloud.backend: "postgres"`: Postgres ledger using `connectionString` or `connectionStringEnv`. + +Daily, monthly, and lifetime budgets require cumulative storage. Per-request budgets do not. Budgets support gateway-key, tenant, and model-alias scopes; hard mode rejects exhausted requests, while soft mode records warnings. See [CLI reference](cli.md) for budget maintenance commands. diff --git a/src/cli/index.ts b/src/cli/index.ts index 0c8a928..85a7968 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -114,18 +114,20 @@ function help(): string { return `Hasna Gateway ${gatewayVersion} Usage: + gateway --help + gateway --version gateway serve --config gateway.config.json [--host 127.0.0.1] [--port 8787] gateway validate --config gateway.config.json gateway smoke --config gateway.config.json [--model fast] gateway smoke --config gateway.config.json --all - gateway budget-add --config gateway.config.json --id daily --window daily [--tenant acme] [--model coding] [--max-usd 1] [--max-total-tokens 100000] + gateway budget-add --config gateway.config.json --id daily [--window daily] [--mode hard] [--gateway-key fingerprint] [--tenant acme] [--model coding] [--max-usd 1] [--max-input-tokens 50000] [--max-output-tokens 50000] [--max-total-tokens 100000] [--warning-threshold 0.8] [--json] gateway budget-list --config gateway.config.json [--json] - gateway budget-remaining --config gateway.config.json [--id daily] [--json] [--contract] - gateway budget-reset --config gateway.config.json --id daily - gateway route --config gateway.config.json --model coding [--json] [--contract] + gateway budget-remaining --config gateway.config.json [--id daily] [--tenant acme] [--model coding] [--json] [--contract] + gateway budget-reset --config gateway.config.json --id daily [--json] + gateway route --config gateway.config.json --model coding [--stream] [--json] [--contract] gateway routes --config gateway.config.json [--json] [--contract] - gateway uninstall --config gateway.config.json --yes - gateway remove --config gateway.config.json --all --yes + gateway uninstall --config gateway.config.json --yes [--json] + gateway remove --config gateway.config.json --all --yes [--json] gateway help `; } @@ -134,11 +136,16 @@ export async function runCli(argv = process.argv.slice(2)): Promise { const parsed = parseArgs(argv); const configPath = flagString(parsed.flags, "config", "gateway.config.json"); - if (parsed.command === "help" || parsed.flags.help) { + if (parsed.command === "help" || parsed.command === "--help" || parsed.flags.help) { console.log(help()); return; } + if (parsed.command === "--version") { + console.log(gatewayVersion); + return; + } + if (parsed.command === "budget-add") { const raw = await readRawConfig(configPath); const budget = { diff --git a/tests/cli-help.test.ts b/tests/cli-help.test.ts new file mode 100644 index 0000000..0f531f6 --- /dev/null +++ b/tests/cli-help.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test"; +import { gatewayVersion } from "../src/version"; + +function runGateway(args: string[]) { + return Bun.spawnSync({ + cmd: ["bun", "run", "src/cli/index.ts", ...args], + stdout: "pipe", + stderr: "pipe", + env: { ...process.env }, + }); +} + +function text(bytes: Uint8Array): string { + return Buffer.from(bytes).toString("utf-8"); +} + +describe("gateway CLI help", () => { + test("supports standard top-level help and version flags", () => { + const help = runGateway(["--help"]); + expect(help.exitCode).toBe(0); + expect(text(help.stdout)).toContain("gateway budget-add"); + expect(text(help.stdout)).toContain("--max-input-tokens"); + expect(text(help.stderr)).toBe(""); + + const version = runGateway(["--version"]); + expect(version.exitCode).toBe(0); + expect(text(version.stdout).trim()).toBe(gatewayVersion); + expect(text(version.stderr)).toBe(""); + }); +}); From a1423985a2520dd439486f973e787f2f461808a4 Mon Sep 17 00:00:00 2001 From: Andrei Hasna Date: Wed, 29 Jul 2026 18:11:27 +0300 Subject: [PATCH 2/3] docs: correct README claim that cache hits are metered A response-cache hit returns from createChatCompletion immediately after assertBudgetPreflight, so it never reaches normalizeUsage, rateLimit.onUsage, evaluateBudgetPostflight, or the usage ledger append. The README said budget and rate-limit accounting still runs on cache hits, which would lead an operator to believe cached traffic is metered and capped under a hard budget. It is neither. State what actually happens: the budget preflight check and the per-gateway-key request rate limit still run, but a cache hit writes no ledger record and consumes neither budget nor token rate limit quota. Add a regression test pinning that behaviour so the doc claim stays verifiable: four identical requests under an enabled cache, a JSONL ledger, and a hard lifetime maxTotalTokens budget produce one provider call, one ledger line, one onUsage event, and unchanged remaining budget. --- README.md | 2 +- tests/gateway.test.ts | 61 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f2fcc7a..f1e0942 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Providers can use `baseUrl`, `baseUrlEnv`, `apiKeyEnv`, custom `auth`, and stati Smart routing is available with route mode `smart` or request `gateway.routing: "smart"`. It filters by policy first, then scores eligible candidates using configured prices, context, capabilities, quality/latency/success/throughput hints, and deterministic fallback ordering when metrics are missing. -Optional non-streaming chat response caching is configured under `server.responseCache`. Cache entries are in-memory and process-local, only successful responses are cached, budget and rate-limit accounting still runs on cache hits, and clients can bypass lookup with the configured `bypassHeader`. Streaming and embeddings requests are never cached. +Optional non-streaming chat response caching is configured under `server.responseCache`. Cache entries are in-memory and process-local, only successful responses are cached, and clients can bypass lookup with the configured `bypassHeader`. Streaming and embeddings requests are never cached. A cache hit still runs the budget preflight check and still consumes the per-gateway-key request rate limit, but it does not append a usage ledger record and does not consume budget or token rate limit quota. ```json { diff --git a/tests/gateway.test.ts b/tests/gateway.test.ts index 3d36929..5109fd6 100644 --- a/tests/gateway.test.ts +++ b/tests/gateway.test.ts @@ -1,5 +1,6 @@ import { unlink } from "node:fs/promises"; import { describe, expect, test } from "bun:test"; +import { getBudgetStatuses } from "../src/budget"; import { GatewayHttpError } from "../src/errors"; import { createChatCompletion, @@ -7,6 +8,7 @@ import { createEmbeddings, providerErrorMessageFromBody, } from "../src/gateway"; +import type { GatewayUsage } from "../src/types"; import { testConfig, jsonResponse } from "./helpers"; const env = { @@ -465,6 +467,65 @@ describe("chat completion lifecycle", () => { expect(second.body.id).toBe("provider-cached-1"); }); + test("does not meter cache hits into the ledger, budget, or token rate limit", async () => { + const path = `/tmp/hasna-gateway-cache-accounting-${crypto.randomUUID()}.jsonl`; + const config = testConfig(); + config.storage.usageLedgerPath = path; + config.server.responseCache = { + ...config.server.responseCache, + enabled: true, + ttlMs: 60_000, + }; + config.budgets = [ + { + id: "tenant-lifetime", + window: "lifetime", + mode: "hard", + scope: { tenant: "tenant-a" }, + maxTotalTokens: 20, + }, + ]; + let callCount = 0; + const fetchImpl = async (): Promise => { + callCount += 1; + return providerResponse(`accounted-${callCount}`); + }; + const rateLimitedUsage: number[] = []; + const request = { + model: "coding", + messages: [{ role: "user" as const, content: "same prompt" }], + }; + const options = { + config, + env, + fetchImpl, + budgetContext: { gatewayKey: "key-a", tenant: "tenant-a" }, + rateLimit: { + onUsage: (usage: GatewayUsage) => { + rateLimitedUsage.push(usage.totalTokens); + }, + }, + }; + + for (let attempt = 0; attempt < 4; attempt += 1) { + await createChatCompletion(options, request); + } + + const ledgerLines = (await Bun.file(path).text()).trim().split("\n").filter(Boolean); + const [budgetStatus] = await getBudgetStatuses(config, { + tenant: "tenant-a", + requestedModel: "coding", + selectedModel: "openai/gpt-4.1-mini", + }); + + expect(callCount).toBe(1); + expect(ledgerLines).toHaveLength(1); + expect(rateLimitedUsage).toEqual([15]); + expect(budgetStatus?.spent.totalTokens).toBe(15); + expect(budgetStatus?.remaining.totalTokens).toBe(5); + await unlink(path); + }); + test("normalizes message object key order for cache keys", async () => { const config = testConfig(); config.server.responseCache = { From 4311a6aa0a0010a38d0465bbe759aa8974957c80 Mon Sep 17 00:00:00 2001 From: Andrei Hasna Date: Wed, 29 Jul 2026 18:25:13 +0300 Subject: [PATCH 3/3] docs: correct embeddings fallback and baseUrl precedence claims The embeddings section of the API contract claimed that providers without an embeddings adapter are skipped. Route filtering only checks the model's `embeddings` capability (src/router.ts routeRejectionReason), never the adapter, so such a candidate is selected and then throws a non-retryable 400 provider_embeddings_unsupported that aborts the request instead of continuing to the next fallback candidate. Document the real behavior and the ordering requirement it implies, including dynamic passthrough ids. The configuration reference stated that `baseUrlEnv` overrides `baseUrl`; providerBaseUrl() returns `provider.baseUrl` first and only falls back to the env var, so an operator following the old text would silently keep sending traffic to the original base URL. Add characterization tests pinning both behaviors so the docs cannot drift again. --- docs/api-contract.md | 2 +- docs/configuration.md | 2 +- tests/gateway.test.ts | 66 ++++++++++++++++++++++++++++++++++++++++++ tests/provider.test.ts | 36 +++++++++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) diff --git a/docs/api-contract.md b/docs/api-contract.md index 268a97a..739bb17 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -128,7 +128,7 @@ Successful non-streaming chat responses can use the optional process-local respo Accepts OpenAI-compatible `model` and `input` fields. `input` may be a string, string array, token array, or array of token arrays. Optional forwarded fields are `encoding_format`, `dimensions`, and `user`. The gateway-only `gateway` field may narrow route policy but is stripped before provider forwarding. -Embedding route candidates must declare the `embeddings` capability. The gateway applies the same auth, policy, budget, fallback, rate-limit, usage-ledger, and metadata rules as non-streaming chat requests. Providers without an embeddings adapter are skipped. Streaming and response caching do not apply. +Embedding route candidates must declare the `embeddings` capability. The gateway applies the same auth, policy, budget, fallback, rate-limit, usage-ledger, and metadata rules as non-streaming chat requests. Route filtering checks the model capability only, never the provider adapter, so a candidate whose adapter cannot embed is selected rather than skipped and then fails the whole request with a non-retryable `400 provider_embeddings_unsupported`; remaining fallback candidates are not attempted. Only the OpenAI-compatible adapter implements embeddings today, so order embeddings routes so that every eligible candidate is served by that adapter. Dynamic `provider/model` passthrough ids get the `embeddings` capability synthesized for any provider and fail the same way. Streaming and response caching do not apply. ```json { diff --git a/docs/configuration.md b/docs/configuration.md index ee9c979..fdd06ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -57,7 +57,7 @@ Omitted top-level values normalize to these local-first defaults: ## Providers And Models -Providers declare `id`, `displayName`, `kind`, endpoint/auth settings, regions, and data policy. `baseUrlEnv` overrides `baseUrl`. `auth` supports bearer, custom-header, and no-auth modes; `headers` supports static values and env-derived values with optional prefixes and required markers. +Providers declare `id`, `displayName`, `kind`, endpoint/auth settings, regions, and data policy. `baseUrl` takes precedence; `baseUrlEnv` is read only when `baseUrl` is absent, and config validation requires one of the two. `auth` supports bearer, custom-header, and no-auth modes; `headers` supports static values and env-derived values with optional prefixes and required markers. Models declare a stable gateway `id`, `providerId`, upstream `providerModel`, aliases, and capabilities. Optional context, price, quality, latency, success-rate, and throughput values drive policy filters and smart scoring. The `embeddings` capability is required for embedding routes. diff --git a/tests/gateway.test.ts b/tests/gateway.test.ts index 5109fd6..e444c9a 100644 --- a/tests/gateway.test.ts +++ b/tests/gateway.test.ts @@ -1046,4 +1046,70 @@ describe("chat completion lifecycle", () => { expect(ledgerText).toContain('"totalTokens":2'); await unlink(path); }); + + test("fails the request when the routed embeddings adapter cannot embed instead of falling back", async () => { + const config = testConfig(); + config.providers.push({ + id: "anthropic", + displayName: "Anthropic", + kind: "anthropic", + baseUrl: "https://api.anthropic.test", + apiKeyEnv: "ANTHROPIC_API_KEY", + enabled: true, + regions: ["us"], + dataPolicy: { + allowTraining: false, + allowLogging: false, + byokOnly: true, + zeroDataRetentionAvailable: false, + }, + }); + config.models.push({ + id: "anthropic/embed-v1", + providerId: "anthropic", + providerModel: "embed-v1", + aliases: ["embeddings"], + capabilities: ["embeddings"], + }); + const route = config.routes.find((entry) => entry.id === "embeddings"); + expect(route).toBeDefined(); + route!.fallbackModelIds = ["anthropic/embed-v1", "openai/text-embedding-3-small"]; + + const calls: string[] = []; + let thrown: unknown; + try { + await createEmbeddings( + { + config, + env: { + GATEWAY_API_KEY: "gateway", + OPENAI_API_KEY: "openai", + ANTHROPIC_API_KEY: "anthropic", + }, + fetchImpl: async (url: string | URL | Request): Promise => { + calls.push(String(url)); + return jsonResponse({ + object: "list", + data: [{ object: "embedding", index: 0, embedding: [0.1] }], + usage: { prompt_tokens: 1, total_tokens: 1 }, + }); + }, + }, + { + model: "embeddings", + input: "unreachable fallback", + }, + ); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(GatewayHttpError); + expect(thrown).toMatchObject({ + status: 400, + code: "provider_embeddings_unsupported", + retryable: false, + }); + expect(calls).toEqual([]); + }); }); diff --git a/tests/provider.test.ts b/tests/provider.test.ts index d6d549d..f382359 100644 --- a/tests/provider.test.ts +++ b/tests/provider.test.ts @@ -9,6 +9,7 @@ import { toProviderChatBody, toProviderEmbeddingsBody, } from "../src/providers"; +import { providerBaseUrl } from "../src/provider-config"; import { testConfig } from "./helpers"; describe("OpenAI-compatible provider adapter", () => { @@ -544,3 +545,38 @@ describe("Google Gemini provider adapter", () => { }); }); }); + +describe("provider base URL resolution", () => { + const baseProvider = { + id: "example", + displayName: "Example", + kind: "openai-compatible" as const, + apiKeyEnv: "EXAMPLE_API_KEY", + enabled: true, + regions: ["us"], + dataPolicy: { + allowTraining: false, + allowLogging: false, + byokOnly: true, + zeroDataRetentionAvailable: false, + }, + }; + + test("prefers baseUrl over baseUrlEnv", () => { + const resolved = providerBaseUrl( + { ...baseProvider, baseUrl: "https://api.example.test/v1", baseUrlEnv: "EXAMPLE_BASE_URL" }, + { EXAMPLE_BASE_URL: "https://proxy.example.test/v1" }, + ); + + expect(resolved).toBe("https://api.example.test/v1"); + }); + + test("falls back to baseUrlEnv only when baseUrl is absent", () => { + const resolved = providerBaseUrl( + { ...baseProvider, baseUrlEnv: "EXAMPLE_BASE_URL" }, + { EXAMPLE_BASE_URL: "https://proxy.example.test/v1" }, + ); + + expect(resolved).toBe("https://proxy.example.test/v1"); + }); +});