diff --git a/.amplifier/digital-twin-universe/profiles/e2e.yaml b/.amplifier/digital-twin-universe/profiles/e2e.yaml index 990c0b8d..2281bbec 100644 --- a/.amplifier/digital-twin-universe/profiles/e2e.yaml +++ b/.amplifier/digital-twin-universe/profiles/e2e.yaml @@ -9,6 +9,14 @@ # GITEA_URL e.g. http://localhost:10110 (Gitea mirror base) # GITEA_TOKEN admin token for the url_rewrites proxy auth # AA_E2E_BASE_IMAGE container image (a pre-baked image can replace the stock one) +# VLLM_BASE_URL vllm suite only; e.g. http://localhost:8007/v1 (may be empty) +# VLLM_MODEL vllm suite only; model id to pin (may be empty) +# VLLM_API_KEY vllm suite only; usually empty for a local server +# +# The VLLM_* trio are vars, not passthrough entries, precisely because they must be +# localhost-rewritten: the vLLM server runs on the host, and DTU rewrites localhost -> +# bridge gateway IP in var values but copies passthrough values verbatim. See +# tests/e2e/framework/dtu_manager.py::_build_varmap. base: image: "${AA_E2E_BASE_IMAGE}" @@ -46,7 +54,13 @@ provision: files: - { src: ./dtu/install-amplifier-agent.sh, dest: /root/e2e/install-amplifier-agent.sh } - { src: ./dtu/host-config.json, dest: /root/e2e/host-config.json } + - { src: ./dtu/setup-vllm-env.sh, dest: /root/e2e/setup-vllm-env.sh } setup_cmds: + # Only the vllm suite needs this, and it is a no-op for everyone else: with the + # vars unset the script writes empty exports, the provider reports source "none", + # and the suite skips itself. Runs before the install so the environment is in + # place for anything the install step might consult. + - "bash /root/e2e/setup-vllm-env.sh '${VLLM_BASE_URL}' '${VLLM_MODEL}' '${VLLM_API_KEY}'" # Insurance for the GitHub Copilot provider, which spawns a Node-based Copilot CLI. # DTU exports SSL_CERT_FILE / REQUESTS_CA_BUNDLE / UV_NATIVE_TLS but never # NODE_EXTRA_CA_CERTS, so a Node process would not trust the interception CA. diff --git a/CHANGELOG.md b/CHANGELOG.md index cec3b959..323b147e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 env read also accepts `GEMINI_API_KEY`, `GOOGLE_API_KEY` taking precedence when both are set), and `auth set gemini` is accepted and stores the key like any other keyed provider. Default model is `gemini-2.5-flash`. +- **vLLM provider.** `provider.module: "vllm"` is now a valid host-config value, backed + by `amplifier-module-provider-vllm`. It integrates a self-hosted or remote vLLM server + for open-weight models (e.g. gpt-oss), talking vLLM's OpenAI-compatible **Responses + API** (`/v1/responses`) rather than the Chat Completions wire — distinct from + `chat-completions`, and a sibling of `openai`/`azure-openai` on the wire shape, while + remaining endpoint-agnostic like `chat-completions`. Supports reasoning models, + reasoning-block separation, and tool calling. Its credential is an endpoint, not a key: + `VLLM_BASE_URL` (required) selects the server, and `VLLM_API_KEY` (optional) is sent + only when set, since a local vLLM server commonly needs none. When it is unset, the + same placeholder the provider module itself defaults to is supplied, so `run` and + `models list --provider vllm` both work against a keyless server; a key set in host + config's `provider.config` still takes precedence over that placeholder. Both + variables are environment-only; the persisted credentials file is not consulted for + this provider. ### Fixed diff --git a/README.md b/README.md index be3b504a..6c2afb6b 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Public integrations run opencode, paperclip, and NanoClaw on it: see [who has in `amplifier-agent` ships with: -- Eight providers behind one interface: Anthropic, OpenAI, Azure OpenAI, Ollama, GitHub Copilot, ChatGPT (a Plus/Pro/Team subscription via OAuth device-code, no API key), Chat Completions (any OpenAI Chat Completions-compatible endpoint, e.g. llama.cpp, vLLM, LM Studio), and Gemini (Google's Gemini API, large context windows plus thinking/reasoning support), with credentials read from the environment or a cached OAuth session +- Nine providers behind one interface: Anthropic, OpenAI, Azure OpenAI, Ollama, GitHub Copilot, ChatGPT (a Plus/Pro/Team subscription via OAuth device-code, no API key), Chat Completions (any OpenAI Chat Completions-compatible endpoint, e.g. llama.cpp, vLLM, LM Studio), Gemini (Google's Gemini API, large context windows plus thinking/reasoning support), and vLLM (a self-hosted or remote vLLM server via its OpenAI-compatible Responses API, for open-weight models like gpt-oss), with credentials read from the environment or a cached OAuth session - Role-based model routing, so a sub-agent gets a model matched to its job rather than the frontier model for everything, re-matched when you switch providers - Context management that keeps long sessions running, compacting history before it overruns the window - Tools for filesystem, bash, web, search, todo, and MCP diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index cae3227b..27594b0b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -12,14 +12,15 @@ Provider is auto-detected from environment variables in this precedence: 4. `OLLAMA_HOST` (defaults to `http://localhost:11434`) 5. `GOOGLE_API_KEY` (`GEMINI_API_KEY` is also accepted by the provider module, `GOOGLE_API_KEY` takes precedence) -`github-copilot`, `openai-chatgpt`, and `chat-completions` are excluded from this auto-detect chain --- none of them resolves from a single API-key environment variable. `github-copilot` reads its own -token chain from the environment (see below). `openai-chatgpt` has no credential env var at all: it +`github-copilot`, `openai-chatgpt`, `chat-completions`, and `vllm` are excluded from this auto-detect +chain -- none of them resolves from a single API-key environment variable. `github-copilot` reads its +own token chain from the environment (see below). `openai-chatgpt` has no credential env var at all: it authenticates via OAuth device-code, caching tokens to `~/.amplifier/openai-chatgpt-oauth.json`. `chat-completions` needs an endpoint rather than a key (`CHAT_COMPLETIONS_BASE_URL`, plus optional -`CHAT_COMPLETIONS_API_KEY`) and has no implicit default endpoint to fall back to. All three must be -selected explicitly with `provider.module` in a host config file, rather than silently winning a -"first match" race. +`CHAT_COMPLETIONS_API_KEY`) and has no implicit default endpoint to fall back to. `vllm` is the same +shape, pointed at a self-hosted or remote vLLM server instead: `VLLM_BASE_URL` (required), plus +optional `VLLM_API_KEY`, with no implicit default endpoint either. All four must be selected explicitly +with `provider.module` in a host config file, rather than silently winning a "first match" race. Override by passing `--config ` at a host config file that names a provider explicitly. @@ -85,6 +86,13 @@ This matters for hosts that spawn `amplifier-agent` as a subprocess: once you ha > # export CHAT_COMPLETIONS_API_KEY=... # only if your server requires one > ``` +> **`vllm` is the same shape as `chat-completions`, environment-only for the same reason.** Its "credential" is the target `base_url` -- the self-hosted or remote vLLM server to talk to -- not an API key: `VLLM_BASE_URL` (required) and `VLLM_API_KEY` (optional; a local vLLM server needs none, and both the module and the agent fall back to the placeholder `"EMPTY"` when it is unset, so a keyless server works for both `run` and `models list`). The persisted credentials file is not consulted for this provider either. Unlike `chat-completions`, which speaks the OpenAI Chat Completions wire, `vllm` talks vLLM's OpenAI-compatible **Responses API**, and supports reasoning models, reasoning-block separation, and tool calling. +> +> ```bash +> export VLLM_BASE_URL=http://localhost:8000/v1 +> # export VLLM_API_KEY=... # only if your server requires one +> ``` + The file format is a versioned JSON envelope: ```jsonc diff --git a/docs/E2E_TESTING.md b/docs/E2E_TESTING.md index e741253d..c1cacc2e 100644 --- a/docs/E2E_TESTING.md +++ b/docs/E2E_TESTING.md @@ -78,6 +78,31 @@ snapshot-at-launch reason. `GOOGLE_API_KEY` is the canonical variable even thoug Google GenAI SDK also accepts `GEMINI_API_KEY`: it takes precedence, and it is the one `providers list` and `models list` consult. +`VLLM_BASE_URL` is optional and only the `vllm` suite uses it. It names a vLLM server you +are running yourself, so that suite skips unless you point it at one — and skips again if +the endpoint is set but not answering, since a server being down is a fact about your +machine rather than a defect in amplifier-agent. + +```bash +export VLLM_BASE_URL=http://localhost:8007/v1 # required to run the suite +export VLLM_MODEL=your-org/your-model # optional; see below +export VLLM_API_KEY=... # optional; local servers rarely need one +uv run python tests/e2e/framework/cli.py run vllm +``` + +Write the URL exactly as you would use it on the host: `localhost` is rewritten to the +container bridge gateway IP at launch, because inside the DTU `localhost` is the container +itself. That rewriting is why these three travel as DTU `--var` values rather than +`passthrough` entries — passthrough copies host values verbatim, and a verbatim +`localhost` would resolve to the wrong machine. The plumbing is +`dtu_manager._build_varmap()` plus `provisioning/setup-vllm-env.sh`, which also exempts +the vLLM host from the interception proxy so streaming is not buffered. + +`VLLM_MODEL` is optional. When unset, the suite uses the first model id the server +advertises on `/v1/models`, which is the right answer for a single-model vLLM process. +Set it when your server hosts more than one. Your server must bind `0.0.0.0` rather than +`127.0.0.1`, or the container cannot reach it. + ## Running ```bash diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 655bd920..a6ce9c54 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -8,7 +8,7 @@ The engine runs **one turn per invocation** and exits. Continuity across turns c `amplifier-agent` is a standalone binary. You do not need the Amplifier CLI, bundles, or any other repository in the `microsoft/amplifier*` family, and none of them is a substitute for it here. -Use it when your software needs to run an agent: a loop with tools, file access, sub-agents, and/or multi-turn state. It also works for plain LLM calls, where you get routing across eight providers behind one interface. +Use it when your software needs to run an agent: a loop with tools, file access, sub-agents, and/or multi-turn state. It also works for plain LLM calls, where you get routing across nine providers behind one interface. Then pick a surface below, install the engine ([INSTALL.md](INSTALL.md)), and finish with the [checklist](#checklist-for-a-new-integration). diff --git a/docs/LAYERS_AND_RELEASES.md b/docs/LAYERS_AND_RELEASES.md index 1aa51841..50caab55 100644 --- a/docs/LAYERS_AND_RELEASES.md +++ b/docs/LAYERS_AND_RELEASES.md @@ -115,7 +115,7 @@ The engine ships with `bundle.md` baked into the wheel. It declares which module **Pre-wired modules:** -- **Providers:** `provider-anthropic`, `provider-openai`, `provider-azure-openai`, `provider-ollama`, `provider-github-copilot`, `provider-openai-chatgpt`, `provider-chat-completions`, `provider-gemini` +- **Providers:** `provider-anthropic`, `provider-openai`, `provider-azure-openai`, `provider-ollama`, `provider-github-copilot`, `provider-openai-chatgpt`, `provider-chat-completions`, `provider-gemini`, `provider-vllm` - **Orchestrator:** `loop-streaming` (with `extended_thinking: true`) - **Context:** `context-simple` (300K tokens, auto-compact at 80%) - **Tools:** `tool-filesystem`, `tool-bash`, `tool-web`, `tool-search`, `tool-todo`, `tool-apply-patch`, `tool-delegate`, `tool-mcp`, `tool-skills`, `tool-mode`, `tool-recipes` diff --git a/docs/architecture/architecture.dot b/docs/architecture/architecture.dot index 30a2de4a..d9f09d38 100644 --- a/docs/architecture/architecture.dot +++ b/docs/architecture/architecture.dot @@ -84,7 +84,7 @@ digraph amplifier_agent { node [fillcolor="#d9d9d9"]; kernel [label="amplifier-foundation\nbundle / session kernel"]; - providers [label="LLM providers\nanthropic · openai · azure\nollama · copilot · chatgpt\nchat-completions · gemini"]; + providers [label="LLM providers\nanthropic · openai · azure\nollama · copilot · chatgpt\nchat-completions · gemini · vllm"]; mcp [label="MCP servers"]; } diff --git a/docs/spec/bundle-and-cache.md b/docs/spec/bundle-and-cache.md index 073dc647..1dcac8bf 100644 --- a/docs/spec/bundle-and-cache.md +++ b/docs/spec/bundle-and-cache.md @@ -31,7 +31,7 @@ default_provider: anthropic REQUIRED, engine-level, top-level key providers: install-only stubs, no config and no credentials provider-anthropic, provider-openai, provider-azure-openai, provider-ollama, provider-github-copilot, provider-openai-chatgpt, - provider-chat-completions, provider-gemini + provider-chat-completions, provider-gemini, provider-vllm session.orchestrator: loop-streaming extended_thinking: true session.context: context-simple max_tokens 300000, auto_compact diff --git a/docs/spec/host-config.md b/docs/spec/host-config.md index d681ae87..5a199a48 100644 --- a/docs/spec/host-config.md +++ b/docs/spec/host-config.md @@ -82,7 +82,7 @@ approval.patterns must be a list of strings ``` provider.module one of: anthropic, openai, azure-openai, ollama, github-copilot, - openai-chatgpt, chat-completions, gemini + openai-chatgpt, chat-completions, gemini, vllm provider.config free-form; belongs to the provider module ``` @@ -102,7 +102,7 @@ module config. Closed per-entry schema: } ``` -`module` defaults to the entry's own id when omitted and must be one of the eight valid module names. +`module` defaults to the entry's own id when omitted and must be one of the nine valid module names. `config` must be an object. Unknown keys inside an entry raise `config_unknown_key`. An empty `providers` object passes validation; HTTP startup rejects it separately at boot so single-turn mode never trips on a stale block. diff --git a/docs/spec/providers-and-models.md b/docs/spec/providers-and-models.md index 2d8a144b..637b36e7 100644 --- a/docs/spec/providers-and-models.md +++ b/docs/spec/providers-and-models.md @@ -9,7 +9,7 @@ routes a wire `model` field to a provider (see `http-face.md`). ## Supported providers -Eight providers are supported, and only eight. The provider name is the value used in configuration, +Nine providers are supported, and only nine. The provider name is the value used in configuration, in `auth` subcommands, and in `models list --provider`. ``` @@ -21,10 +21,11 @@ github-copilot provider-github-copilot openai-chatgpt provider-openai-chatgpt chat-completions provider-chat-completions gemini provider-gemini +vllm provider-vllm ``` Each module is installed from `git+https://github.com/microsoft/amplifier-module-@main`. -All eight are declared by the shipped bundle (`bundle.md`'s top-level `providers:` stub list) as +All nine are declared by the shipped bundle (`bundle.md`'s top-level `providers:` stub list) as install-only, so preparing the bundle makes every provider importable before any session exists. The agent holds no static table of default models, credential field shapes, or display names. Those @@ -54,6 +55,7 @@ github-copilot GITHUB_TOKEN openai-chatgpt (none -- OAuth device-code) chat-completions CHAT_COMPLETIONS_BASE_URL, plus optional CHAT_COMPLETIONS_API_KEY gemini GOOGLE_API_KEY +vllm VLLM_BASE_URL (required), plus optional VLLM_API_KEY ``` `AZURE_OPENAI_KEY` is the only deprecated alias. Consulting it emits a one-time warning on stderr. @@ -82,6 +84,23 @@ chat-completions -- with no `CHAT_COMPLETIONS_BASE_URL` in the environment it re unconditionally to `source == "none"`, with no file fallback and no usable default to fall back to (unlike ollama's built-in localhost). +vllm has the same shape of dedicated resolution branch, for the same reason: `VLLM_BASE_URL` +lands in `fields["base_url"]`, and `VLLM_API_KEY` lands in `fields["api_key"]` alongside it. +`VLLM_API_KEY` differs from chat-completions' optional key in one way: when it is unset, +`fields["api_key"]` is still populated, with the same `"EMPTY"` placeholder the provider module +defaults to. A local vLLM server commonly needs no auth, but the OpenAI SDK the module wraps +still requires some value, and `models list` builds the provider straight from these fields +rather than through the module's `mount()` — so omitting the field entirely would hand that +path an empty key and fail against a keyless server. The placeholder is not a credential: a +key supplied through host config's `provider.config` takes precedence over it, while a real +`VLLM_API_KEY` from the environment is re-asserted over host config as usual. The persisted +credentials file +is never consulted for vllm either -- with no `VLLM_BASE_URL` in the environment it resolves +unconditionally to `source == "none"`, with no file fallback and no usable default to fall back +to. The distinction from chat-completions is the wire, not the credential shape: vllm targets +vLLM's OpenAI-compatible **Responses API** (`/v1/responses`), not the Chat Completions API, which +is what lets it support reasoning models, reasoning-block separation, and tool calling. + gemini lists only `GOOGLE_API_KEY` here. The Google GenAI SDK also accepts `GEMINI_API_KEY` (`GOOGLE_API_KEY` takes precedence when both are set), and the provider module's own env read honours that; listing `GEMINI_API_KEY` in this table would mark it deprecated, which it is not. @@ -95,9 +114,10 @@ host reports unresolved on purpose, so auto-enrollment does not enlist a local d be running. Requesting credentials for a key-based provider that resolves to `none` is an error. Ollama, -chat-completions, and unrecognized provider names never raise -- chat-completions is not one of -the key-based providers this rule applies to, so a missing `CHAT_COMPLETIONS_BASE_URL` is left for -the provider module itself to reject at call time, not raised here. +chat-completions, vllm, and unrecognized provider names never raise -- chat-completions and vllm +are not among the key-based providers this rule applies to, so a missing `CHAT_COMPLETIONS_BASE_URL` +or `VLLM_BASE_URL` is left for the respective provider module to reject at call time, not raised +here. ## The credentials file @@ -143,6 +163,10 @@ above the chat-completions resolution branch never reads the credentials file -- `CHAT_COMPLETIONS_BASE_URL` / `CHAT_COMPLETIONS_API_KEY` in the environment are consulted. Set those instead of relying on `auth set` for this provider. +`auth set vllm` is likewise accepted and likewise ignored at resolution time -- only +`VLLM_BASE_URL` / `VLLM_API_KEY` in the environment are consulted. Set those instead of relying on +`auth set` for this provider. + `auth clear` without `--force` exits 2. ## Provider selection at boot @@ -153,7 +177,7 @@ those instead of relying on `auth set` for this provider. 3. no further fallback: a bundle declaring neither is a hard error at boot ``` -`provider.module` is closed to the eight supported names. Any other value fails validation with +`provider.module` is closed to the nine supported names. Any other value fails validation with error code `config_invalid_provider_module`. `"auto"` is not a valid value. There is no `--provider` flag and no environment-based provider auto-detection. See Non-goals. diff --git a/skills/amplifier-agent/SKILL.md b/skills/amplifier-agent/SKILL.md index d46615b1..8d7a276b 100644 --- a/skills/amplifier-agent/SKILL.md +++ b/skills/amplifier-agent/SKILL.md @@ -22,7 +22,7 @@ metadata: `amplifier-agent` is an agent engine that other software runs on. Give it a prompt and it runs the full loop, with tools, sub-agents, skills, and MCP, then returns a result. Anything that can spawn a subprocess can use it; Python hosts can embed the engine library in-process instead. -Reach for it when the project needs an *agent* (a tool loop, file access, sub-agents, multi-turn state) rather than a single completion. You can also use it for plain LLM calls, with routing across eight providers behind one interface. +Reach for it when the project needs an *agent* (a tool loop, file access, sub-agents, multi-turn state) rather than a single completion. You can also use it for plain LLM calls, with routing across nine providers behind one interface. **The engine runs one turn per invocation and exits.** Continuity across turns comes from a session id, not from a long-lived process. Every surface below is a different way of delivering a prompt to that same engine. @@ -54,7 +54,7 @@ The installer needs `uv` and `curl` and will not bootstrap them silently; it tel Install as **the same user that runs the host process**; a host spawning a subprocess inherits that user's `PATH`. `amplifier-agent doctor` is the check that the install actually works, so run it before writing any integration code. -Credentials are read from the environment, first match wins: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY` plus `AZURE_OPENAI_ENDPOINT`, `OLLAMA_HOST`. GitHub Copilot is environment-only (`COPILOT_AGENT_TOKEN`, `COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`). ChatGPT (`openai-chatgpt`) has no credential env var at all: it authenticates via OAuth device-code, caching tokens to `~/.amplifier/openai-chatgpt-oauth.json`. The `chat-completions` provider is environment-only too, for any OpenAI Chat Completions-compatible endpoint (llama.cpp, vLLM, LM Studio, LocalAI, and similar): `CHAT_COMPLETIONS_BASE_URL` (required) plus optional `CHAT_COMPLETIONS_API_KEY`. Or store a static key with `amplifier-agent auth set anthropic sk-ant-...` (not supported for github-copilot or openai-chatgpt). +Credentials are read from the environment, first match wins: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY` plus `AZURE_OPENAI_ENDPOINT`, `OLLAMA_HOST`. GitHub Copilot is environment-only (`COPILOT_AGENT_TOKEN`, `COPILOT_GITHUB_TOKEN`, `GH_TOKEN`, `GITHUB_TOKEN`). ChatGPT (`openai-chatgpt`) has no credential env var at all: it authenticates via OAuth device-code, caching tokens to `~/.amplifier/openai-chatgpt-oauth.json`. The `chat-completions` provider is environment-only too, for any OpenAI Chat Completions-compatible endpoint (llama.cpp, vLLM, LM Studio, LocalAI, and similar): `CHAT_COMPLETIONS_BASE_URL` (required) plus optional `CHAT_COMPLETIONS_API_KEY`. The `vllm` provider is the same shape, for a self-hosted or remote vLLM server via its OpenAI-compatible Responses API: `VLLM_BASE_URL` (required) plus optional `VLLM_API_KEY`. Or store a static key with `amplifier-agent auth set anthropic sk-ant-...` (not supported for github-copilot or openai-chatgpt). ## Pick a surface @@ -191,7 +191,7 @@ A per-instance config file looks like this: | `config_unreadable`, `config_malformed_json` | The `--config` file could not be opened, or is not a JSON object | Check the path the host wrote, and that it serialized an object | | `config_unknown_key` | Unrecognized **top-level** config key | The top level is closed: `approval`, `provider`, `providers`, `mcp`, `skills`, `debug`, `allowProtocolSkew` | | `config_invalid_type` | A known key has the wrong type, or an unknown sub-key in a closed inner shape | `skills.*` and `debug.*` are closed and raise this rather than `config_unknown_key`, which is reserved for the top level and `providers.` entries | -| `config_invalid_provider_module` | `provider.module` is not a known provider | One of `anthropic`, `openai`, `azure-openai`, `ollama`, `github-copilot`, `openai-chatgpt`, `chat-completions`, `gemini`. `"auto"` is not valid | +| `config_invalid_provider_module` | `provider.module` is not a known provider | One of `anthropic`, `openai`, `azure-openai`, `ollama`, `github-copilot`, `openai-chatgpt`, `chat-completions`, `gemini`, `vllm`. `"auto"` is not valid | | `protocol_version_mismatch` | Wrapper and engine protocol versions differ | Update the lagging side. `allowProtocolSkew` is an unblock, not a fix | | `lifecycle_unsupported` | `submit()` called twice on one handle | New handle per turn, same `sessionId` with `resume` | | `env_injection_rejected` | The wrapper refused the environment you asked it to inject | Check the key against the wrapper's allowlist and blocked-key list | diff --git a/src/amplifier_agent_cli/provider_sources.py b/src/amplifier_agent_cli/provider_sources.py index 0bd8b092..4cc271c8 100644 --- a/src/amplifier_agent_cli/provider_sources.py +++ b/src/amplifier_agent_cli/provider_sources.py @@ -114,6 +114,7 @@ def _emit_legacy_env_var_notice(legacy_var: str, preferred_var: str) -> None: "openai-chatgpt", "chat-completions", "gemini", + "vllm", ) @@ -157,9 +158,28 @@ def _emit_legacy_env_var_notice(legacy_var: str, preferred_var: str) -> None: "module": "provider-gemini", "source": "git+https://github.com/microsoft/amplifier-module-provider-gemini@main", }, + "vllm": { + "module": "provider-vllm", + "source": "git+https://github.com/microsoft/amplifier-module-provider-vllm@main", + }, } +#: Placeholder API key for a keyless vLLM server. +#: +#: The provider module's own ``mount()`` defaults ``VLLM_API_KEY`` to this exact +#: string, because a self-hosted vLLM commonly needs no auth but the OpenAI SDK +#: it wraps still insists on *some* value. Mirroring that default here is what +#: keeps ``models list --provider vllm`` working: unlike ``run``, that path +#: builds the provider straight from the resolved credential fields, so an +#: absent key would otherwise reach the constructor as ``""``. See the vllm +#: branch of :func:`resolve_provider_credentials` for the full story. +#: +#: It is a placeholder, not a secret. :func:`build_provider_entry` must not let +#: it overwrite a real key supplied through host config's ``provider.config``. +VLLM_KEYLESS_API_KEY: Final[str] = "EMPTY" + + #: Map provider short-name → ``(primary_env, *legacy_envs)``. #: #: Small auxiliary mapping used by :func:`resolve_credential_detailed` to @@ -474,6 +494,51 @@ def resolve_credential_detailed(provider_name: str) -> CredentialResolution: fields=cc_fields, ) + if provider_name == "vllm": + # vLLM provider: talks the OpenAI Responses API to a self-hosted or + # remote vLLM server. base_url is the required "credential" (which server + # to reach); VLLM_API_KEY is optional -- local vLLM needs no auth (the + # module defaults it to "EMPTY"). Dedicated branch so base_url lands in + # fields["base_url"], not fields["api_key"] (same reasoning as + # chat-completions). Absent base_url is honestly resolved=False/source=none: + # there is no usable default to guess (the module's localhost:8000 fallback + # is not something amplifier-agent should claim is "configured"). + base_url = os.environ.get("VLLM_BASE_URL", "") + if not base_url: + return CredentialResolution( + provider=provider_name, + resolved=False, + source="none", + env_var="VLLM_BASE_URL", + fields={}, + ) + # api_key is ALWAYS carried here, unlike chat-completions above, and falls + # back to the same placeholder the provider module's own mount() uses. + # + # The asymmetry is deliberate. `run` resolves the key inside the module + # (`os.environ.get("VLLM_API_KEY", "EMPTY")`), where *absent* correctly + # yields the default. `models list` does not go through mount() at all: it + # builds the provider directly from these fields via + # _try_instantiate_provider, which falls back to ``api_key=""`` for a field + # that is not present. VLLMProvider's signature is + # ``(base_url, *, api_key="EMPTY", config=...)``, so it matches that + # helper's base_url+api_key+config attempt and receives the empty string, + # overriding its own default -- and the OpenAI SDK rejects an empty key + # with "Missing credentials ... set OPENAI_API_KEY", which names nothing + # the user can act on. Carrying the placeholder makes the two paths agree + # and keeps the common keyless local server working in both. + vllm_fields: dict[str, str] = { + "base_url": base_url, + "api_key": os.environ.get("VLLM_API_KEY", "") or VLLM_KEYLESS_API_KEY, + } + return CredentialResolution( + provider=provider_name, + resolved=True, + source="env", + env_var="VLLM_BASE_URL", + fields=vllm_fields, + ) + env_vars = PROVIDER_CREDENTIAL_VARS.get(provider_name) if not env_vars: return CredentialResolution(provider=provider_name, resolved=False, source="none", env_var=None, fields={}) @@ -692,7 +757,19 @@ def build_provider_entry( config["effort"] = effort_override if extra_config: config.update(extra_config) - _reassert_protected_keys(config, creds=creds, priority=priority) + + # A placeholder credential stands in for the provider module's own default + # (see :data:`VLLM_KEYLESS_API_KEY`), so it is not the kind of engine-resolved + # value protected-key re-assertion exists to defend. Dropping it from the + # re-assertion set when host config supplied a real api_key preserves the + # guarantee the chat-completions branch documents: a key set in + # provider.config must not be silently replaced by a "no key needed" default. + # A genuinely resolved VLLM_API_KEY is not a placeholder and still wins. + protected = dict(creds) + if protected.get("api_key") == VLLM_KEYLESS_API_KEY and (extra_config or {}).get("api_key"): + del protected["api_key"] + + _reassert_protected_keys(config, creds=protected, priority=priority) return {"module": entry["module"], "source": entry["source"], "config": config} diff --git a/src/amplifier_agent_lib/bundle/bundle.md b/src/amplifier_agent_lib/bundle/bundle.md index 041408e5..bbf6c55c 100644 --- a/src/amplifier_agent_lib/bundle/bundle.md +++ b/src/amplifier_agent_lib/bundle/bundle.md @@ -74,6 +74,8 @@ providers: source: git+https://github.com/microsoft/amplifier-module-provider-chat-completions@main - module: provider-gemini source: git+https://github.com/microsoft/amplifier-module-provider-gemini@main + - module: provider-vllm + source: git+https://github.com/microsoft/amplifier-module-provider-vllm@main session: raw: true diff --git a/src/amplifier_agent_lib/config/loader.py b/src/amplifier_agent_lib/config/loader.py index 3d433a64..a1df425e 100644 --- a/src/amplifier_agent_lib/config/loader.py +++ b/src/amplifier_agent_lib/config/loader.py @@ -30,7 +30,17 @@ _VALID_TOP_LEVEL_KEYS = frozenset({"mcp", "approval", "provider", "providers", "allowProtocolSkew", "skills", "debug"}) _VALID_PROVIDER_MODULES = frozenset( - {"anthropic", "openai", "azure-openai", "ollama", "github-copilot", "openai-chatgpt", "chat-completions", "gemini"} + { + "anthropic", + "openai", + "azure-openai", + "ollama", + "github-copilot", + "openai-chatgpt", + "chat-completions", + "gemini", + "vllm", + } ) # G3: explicit set of host-supplied approval modes. ``CliApprovalSystem`` accepts # exactly these three strings; any other value must be rejected at parse time diff --git a/tests/e2e/framework/dtu_manager.py b/tests/e2e/framework/dtu_manager.py index 9f5f08c9..e46a6a3e 100644 --- a/tests/e2e/framework/dtu_manager.py +++ b/tests/e2e/framework/dtu_manager.py @@ -42,11 +42,28 @@ def _mirror_repos(gitea: dict[str, Any]) -> list[str]: def _build_varmap(gitea: dict[str, Any]) -> dict[str, str]: - """Assemble the --var map for launch/update.""" + """Assemble the --var map for launch/update. + + The ``VLLM_*`` entries travel as vars rather than ``passthrough`` entries on + purpose. Passthrough copies the host value verbatim, but the vllm suite targets + a vLLM server running on the HOST, and inside the container ``localhost`` is the + container. DTU rewrites ``localhost`` / ``127.0.0.1`` in *var values* to the + bridge gateway IP at launch, which is the same mechanism ``GITEA_URL`` already + depends on -- so routing the endpoint through a var is what makes + ``VLLM_BASE_URL=http://localhost:8007/v1`` actually resolve to the host server. + + Always emitted, even when unset on the host: unresolved ``${VAR}`` references are + left verbatim by DTU's substitution, so omitting them would leak the literal + string ``${VLLM_BASE_URL}`` into the container environment. An empty value is + correct and unambiguous -- the vllm suite skips on it. + """ return { "GITEA_URL": gitea["gitea_url"], "GITEA_TOKEN": gitea["token"], "AA_E2E_BASE_IMAGE": "ubuntu:24.04", + "VLLM_BASE_URL": os.environ.get("VLLM_BASE_URL", ""), + "VLLM_MODEL": os.environ.get("VLLM_MODEL", ""), + "VLLM_API_KEY": os.environ.get("VLLM_API_KEY", ""), } @@ -82,23 +99,27 @@ def _warn_extra_repos(mirrored: list[str]) -> None: def _check_passthrough_env() -> None: - """Warn about passthrough env vars that are missing on the launching process. + """Warn about suite env vars that are missing on the launching process. - DTU bakes each ``passthrough.services`` value into ``/etc/profile.d/dtu-env.sh`` - at launch with a bare ``if value:`` guard -- an unset var produces no export and - no error, so the failure surfaces much later as an opaque provider auth error. - Warning here turns that into an immediate, actionable message. + Two mechanisms end up in the same place. DTU bakes each ``passthrough.services`` + value into ``/etc/profile.d/dtu-env.sh`` at launch with a bare ``if value:`` guard, + and ``_build_varmap``'s ``VLLM_*`` vars are written by ``setup-vllm-env.sh``. Both + treat an absent value as an empty export rather than an error, so the failure + surfaces much later as an opaque provider auth or connection error. Warning here + turns that into an immediate, actionable message. Deliberately a warning, not a hard failure: these are per-suite requirements, and a missing GITHUB_TOKEN should not block someone running the skills or modes suites. Each suite enforces its own requirement directly, and inside the container, which is what actually matters -- github_copilot fails loud via - ``test_ghcp_token_reaches_dtu``, gemini skips itself via its ``gemini_key`` fixture. + ``test_ghcp_token_reaches_dtu``, gemini skips itself via its ``gemini_key`` fixture, + vllm skips itself via its ``vllm_endpoint`` fixture. """ required = ( ("ANTHROPIC_API_KEY", "most suites will fail"), ("GITHUB_TOKEN", "the github_copilot suite will fail"), ("GOOGLE_API_KEY", "the gemini suite will skip"), + ("VLLM_BASE_URL", "the vllm suite will skip"), ) for var, consequence in required: if not os.environ.get(var): diff --git a/tests/e2e/framework/provisioning/setup-vllm-env.sh b/tests/e2e/framework/provisioning/setup-vllm-env.sh new file mode 100644 index 00000000..fbbebb60 --- /dev/null +++ b/tests/e2e/framework/provisioning/setup-vllm-env.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Export the vllm suite's endpoint into the container environment. +# +# Why this is a provisioning script rather than a `passthrough` entry: passthrough +# copies the host value verbatim (engine.py `_write_env`), but the vLLM server the +# suite targets runs on the HOST. Inside the container `localhost` is the container +# itself, so a verbatim `http://localhost:8007/v1` would resolve to the container's +# own loopback and never reach the server. `--var` values, by contrast, are rewritten +# at launch (engine.py `_rewrite_localhost`) so `localhost` / `127.0.0.1` become the +# bridge gateway IP -- exactly the trick GITEA_URL already relies on. So the endpoint +# travels as a var, and this script lands it in the environment. +# +# Args (all positional, all may be empty): +# $1 VLLM_BASE_URL already localhost-rewritten by DTU +# $2 VLLM_MODEL model id to pin; empty means "let the suite discover it" +# $3 VLLM_API_KEY optional; a local vLLM server usually needs none +# +# An empty value is OMITTED rather than exported as an empty string. That distinction +# is load-bearing, not stylistic. The provider module resolves its key as: +# +# api_key = config.get("api_key") or os.environ.get("VLLM_API_KEY", "EMPTY") +# +# and that "EMPTY" default applies only when the variable is ABSENT. Exporting +# VLLM_API_KEY="" makes it present-but-empty, which defeats the default and hands the +# OpenAI SDK an empty key -- surfacing as "Missing credentials ... set OPENAI_API_KEY", +# a message that points nowhere near the real cause. Omitting the export instead is +# what lets an unauthenticated local vLLM server work at all. +# +# `[ -n "$VAR" ]` cannot tell absent from empty, so the suite's guard reads the same +# either way and nothing is lost by omitting. + +set -euo pipefail + +BASE_URL="${1:-}" +MODEL="${2:-}" +API_KEY="${3:-}" + +DEST="/etc/profile.d/zz-vllm.sh" + +# `zz-` prefix is load-bearing. /etc/profile.d is sourced in alphabetical order and +# DTU writes no_proxy in dtu-env.sh; sorting after it is what lets the no_proxy +# amendment below survive rather than be overwritten. +# +# Written with `if` blocks rather than `[ -n "$X" ] && printf ...` on purpose: under +# `set -e` a trailing short-circuit that evaluates false is the script's exit status, +# so an absent API key would make provisioning fail. +printf '#!/bin/bash\n' >"$DEST" +if [ -n "$BASE_URL" ]; then + printf 'export VLLM_BASE_URL=%q\n' "$BASE_URL" >>"$DEST" +fi +if [ -n "$MODEL" ]; then + printf 'export VLLM_MODEL=%q\n' "$MODEL" >>"$DEST" +fi +if [ -n "$API_KEY" ]; then + printf 'export VLLM_API_KEY=%q\n' "$API_KEY" >>"$DEST" +fi + +# Exempt the vLLM host from the mitmproxy interception proxy. +# +# DTU sets no_proxy="localhost,127.0.0.1,::1" when url_rewrites is active. The vLLM +# endpoint is the bridge gateway IP, which is NOT in that list, so without this the +# traffic would route through mitmproxy -- and per DTU's own comment in _write_env, +# mitmproxy "buffers whole response bodies and destroys SSE / token streaming". The +# agent streams, so this is not a theoretical concern. +# +# Only the host is added, never the port: no_proxy matches on host. +if [ -n "$BASE_URL" ]; then + VLLM_HOST="$(printf '%s' "$BASE_URL" | sed -E 's#^[a-zA-Z]+://##; s#[:/].*$##')" + if [ -n "$VLLM_HOST" ]; then + { + printf 'export no_proxy="${no_proxy:+${no_proxy},}%s"\n' "$VLLM_HOST" + printf 'export NO_PROXY="${NO_PROXY:+${NO_PROXY},}%s"\n' "$VLLM_HOST" + } >>"$DEST" + fi +fi + +chmod +x "$DEST" + +# Report without echoing the API key. +echo "[setup-vllm-env] VLLM_BASE_URL=${BASE_URL:-} VLLM_MODEL=${MODEL:-} api_key=$([ -n "$API_KEY" ] && echo present || echo absent)" diff --git a/tests/e2e/suites/vllm/__init__.py b/tests/e2e/suites/vllm/__init__.py new file mode 100644 index 00000000..bfc1d9a4 --- /dev/null +++ b/tests/e2e/suites/vllm/__init__.py @@ -0,0 +1,3 @@ +"""``vllm`` provider e2e suite.""" + +from __future__ import annotations diff --git a/tests/e2e/suites/vllm/cases.py b/tests/e2e/suites/vllm/cases.py new file mode 100644 index 00000000..cf3183ad --- /dev/null +++ b/tests/e2e/suites/vllm/cases.py @@ -0,0 +1,200 @@ +"""Case definitions for the ``vllm`` provider suite. + +Scope is deliberately narrow, and narrower still than the gemini suite's. PR #130 +registered vllm as the ninth provider: a catalog entry, a dedicated credential +branch keyed on an *endpoint* rather than an API key, an accepted +``provider.module`` value, and an install-only bundle stub. Nothing about the +provider's own behaviour is ours to test -- that belongs to +``amplifier-module-provider-vllm``. So this suite proves the four things +amplifier-agent itself is now responsible for: + + 1. vllm appears in the credential-resolution report with the right module + and the right env var, and resolves from the environment + 2. its models can be listed off the configured server + 3. a host config naming it passes validation + 4. a real session actually runs on it + +Model quality is explicitly out of scope. A self-hosted vLLM server is usually +running a small open-weight model, so the smoke case asks for one literal token +and nothing more. Anything that depended on instruction-following fidelity, +tool-calling competence, or reasoning quality would make this suite a flaky +report on the operator's model choice rather than on amplifier-agent's wiring. + +Assertions are on the public payloads documented in +``docs/spec/providers-and-models.md`` and ``docs/spec/cli.md``, never on log +output or internals. +""" + +from __future__ import annotations + +from typing import Any + +from framework.harness import E2ECase + +#: Provider name as it appears in configuration, ``auth`` subcommands, and +#: ``models list --provider``. See docs/spec/providers-and-models.md. +PROVIDER_ID = "vllm" + +#: Module the catalog maps ``vllm`` onto. +PROVIDER_MODULE = "provider-vllm" + +#: The one variable amplifier-agent consults to decide *which server to talk to*. +#: Unlike every keyed provider, vllm's credential is an endpoint, so this lands in +#: ``fields["base_url"]`` rather than ``fields["api_key"]``. ``VLLM_API_KEY`` is +#: optional and deliberately not asserted on: a local vLLM server needs none, and +#: making its presence part of the contract would break the common setup. +CREDENTIAL_VAR = "VLLM_BASE_URL" + +#: In-DTU seed directory and the host config rendered into it by conftest. +DTU_DIR = "/root/e2e/vllm" +CONFIG_PATH = f"{DTU_DIR}/host-config-vllm.json" + +#: Single literal the smoke prompt asks for. Short, unambiguous, and unlikely +#: to appear by accident in a refusal or an error string. Matched case-insensitively +#: and as a substring, so a small model that wraps it in pleasantries still passes -- +#: the case is proving the round trip completed, not that the model obeys precisely. +PING_TOKEN = "PONG" + + +def _providers_rows(parsed: Any) -> list[dict[str, Any]]: + """Pull the row list out of a ``providers list --json`` payload.""" + assert isinstance(parsed, dict), f"providers list did not emit a JSON object: {parsed!r}" + rows = parsed.get("providers") + assert isinstance(rows, list), f"payload has no 'providers' list: {parsed!r}" + return [r for r in rows if isinstance(r, dict)] + + +def expect_vllm_registered(parsed: Any) -> None: + """vllm is in the credential report, wired to the right module and variable. + + Split into separate assertions because the four failures are genuinely + different problems: not in KNOWN_PROVIDERS, wrong catalog entry, wrong + credential variable, and "the endpoint never reached the DTU". The last one + is the reason this suite needs no separate passthrough-guard test -- this + case already proves the var survived the localhost rewrite into the container. + """ + rows = _providers_rows(parsed) + names = [r.get("name") for r in rows] + + assert PROVIDER_ID in names, ( + f"{PROVIDER_ID!r} missing from `providers list`; it is not in KNOWN_PROVIDERS. Saw: {names}" + ) + + row = next(r for r in rows if r.get("name") == PROVIDER_ID) + + assert row.get("module") == PROVIDER_MODULE, ( + f"{PROVIDER_ID} maps to module {row.get('module')!r}, expected {PROVIDER_MODULE!r}" + ) + assert row.get("env_var") == CREDENTIAL_VAR, ( + f"{PROVIDER_ID} reports credential var {row.get('env_var')!r}, expected {CREDENTIAL_VAR!r}" + ) + assert row.get("resolvable") is True and row.get("source") == "env", ( + f"{PROVIDER_ID} did not resolve from the environment " + f"(resolvable={row.get('resolvable')!r}, source={row.get('source')!r}). " + f"{CREDENTIAL_VAR} is not reaching the DTU even though the suite guard saw it." + ) + + +def expect_vllm_models(parsed: Any) -> None: + """``models list --provider vllm`` returns a usable live listing. + + An empty list is a legal answer for some providers but not for this one: vllm + only resolves at all when an endpoint is configured, and the suite guard has + already proved that endpoint is serving ``/v1/models``. Empty here means the + agent queried something other than the configured server. + """ + assert isinstance(parsed, dict), f"models list did not emit a JSON object: {parsed!r}" + assert parsed.get("schema_version") == 1, f"unexpected schema_version: {parsed.get('schema_version')!r}" + assert parsed.get("provider") == PROVIDER_ID, f"payload reports provider {parsed.get('provider')!r}" + + models = parsed.get("models") + assert isinstance(models, list) and models, ( + f"{PROVIDER_ID} returned no models, but the suite guard reached the server's /v1/models. Payload: {parsed!r}" + ) + missing = [m for m in models if not (isinstance(m, dict) and m.get("id"))] + assert not missing, f"model entries without an 'id': {missing!r}" + + +def expect_config_accepts_vllm(parsed: Any) -> None: + """A host config naming ``vllm`` parses cleanly. + + ``config show`` is a diagnostic command and always exits 0, capturing a + rejection into ``host_config.parse_error`` rather than a non-zero exit. So + the absence of that key is the actual assertion: it is what proves ``vllm`` + is in the closed ``provider.module`` set. + """ + assert isinstance(parsed, dict), f"config show did not emit a JSON object: {parsed!r}" + host_config = parsed.get("host_config") + assert isinstance(host_config, dict), f"payload has no 'host_config' block: {parsed!r}" + + assert "parse_error" not in host_config, ( + f"host config naming provider.module={PROVIDER_ID!r} was rejected: {host_config.get('parse_error')!r}. " + f"Expected it to be accepted (error code config_invalid_provider_module means the " + f"provider is missing from the valid module set)." + ) + + provider_block = (host_config.get("parsed") or {}).get("provider") + assert isinstance(provider_block, dict) and provider_block.get("module") == PROVIDER_ID, ( + f"parsed host config does not report provider.module={PROVIDER_ID!r}: {provider_block!r}" + ) + + +def expect_ping(parsed: Any) -> None: + """The reply mentions the ping token. + + Substring, case-insensitive, and tolerant of surrounding text on purpose. A + small open-weight model may prepend a reasoning preamble or wrap the answer in + a sentence; that is a fact about the operator's model, not a defect in the + provider wiring this suite exists to test. What would genuinely fail is the + token never appearing at all, which is what an auth failure, a wrong endpoint, + or a broken mount actually looks like. + """ + text = str(parsed) + assert PING_TOKEN.lower() in text.lower(), ( + f"expected {PING_TOKEN!r} somewhere in the reply; the session did not complete " + f"against the vLLM server. Got:\n{text}" + ) + + +#: Catalog and credential wiring. No model is invoked. +WIRING: list[E2ECase] = [ + E2ECase( + "vllm-providers-list", + "cli", + ["providers", "list", "--json"], + check=expect_vllm_registered, + ), + E2ECase( + "vllm-config-accepted", + "cli", + ["config", "show", "--config", CONFIG_PATH], + check=expect_config_accepts_vllm, + ), +] + +#: Live query against the configured vLLM server, but no completion. +MODELS: list[E2ECase] = [ + E2ECase( + "vllm-models-list", + "cli", + ["models", "list", "--provider", PROVIDER_ID, "--output", "json"], + check=expect_vllm_models, + ), +] + +#: One real session. The cheapest possible proof that the provider module +#: mounts, reaches the server, and completes end to end. +SMOKE: list[E2ECase] = [ + E2ECase( + "vllm-basic-reply", + "cli", + [ + "run", + "-y", + "--config", + CONFIG_PATH, + f"Reply with exactly the word {PING_TOKEN} and nothing else.", + ], + check=expect_ping, + ), +] diff --git a/tests/e2e/suites/vllm/conftest.py b/tests/e2e/suites/vllm/conftest.py new file mode 100644 index 00000000..e9df4669 --- /dev/null +++ b/tests/e2e/suites/vllm/conftest.py @@ -0,0 +1,145 @@ +"""Suite fixtures: endpoint gating, model discovery, and host-config seeding. + +Like ``gemini`` and unlike ``github_copilot``, this suite SKIPS rather than failing +when it is not configured. It goes further, though: gemini skips only when its key is +absent, whereas this suite also skips when the endpoint is *set but unreachable*. + +The difference is what the two variables mean. A missing API key is a setup mistake +with one cause. A vLLM endpoint is a server someone has to be running -- on their own +hardware, with a GPU, hosting a model they chose. It will be down far more often than +it is misconfigured, and "the operator's server is not up right now" is not a defect +in amplifier-agent. Failing the default ``cli.py run`` for that would make the suite a +report on the state of someone's GPU box. + +So: no endpoint, no run, no failure. Unreachable endpoint, no run, no failure. The +skip messages say exactly which of the two happened and what to do about it. +""" + +from __future__ import annotations + +import json +import shlex +import tempfile +from pathlib import Path +from typing import Any + +import pytest +from framework import dtu + +from suites.vllm.cases import CONFIG_PATH, CREDENTIAL_VAR, DTU_DIR + +#: Probes for the variable's presence inside the container without echoing its value. +#: ``bash -lc`` matters: the export lives in /etc/profile.d/zz-vllm.sh, which only a +#: login shell sources. +_PRESENCE_PROBE = f'if [ -n "${CREDENTIAL_VAR}" ]; then echo endpoint=present; else echo endpoint=missing; fi' + +_SKIP_UNSET = ( + f"{CREDENTIAL_VAR} is not set inside the DTU, so the vllm suite is skipped. " + f"To run it: start a vLLM server, then export {CREDENTIAL_VAR} (e.g. " + f"http://localhost:8007/v1) on the host and re-provision " + f"(`uv run python tests/e2e/framework/cli.py up`). Optionally also export " + f"VLLM_MODEL to pin a model id, and VLLM_API_KEY if your server requires auth. " + f"The value is baked into the container at launch, so exporting it against an " + f"already-running DTU has no effect. `localhost` is rewritten to the bridge " + f"gateway IP automatically -- write the URL as you would use it on the host." +) + + +def _skip_unreachable(base_url: str, detail: str) -> str: + return ( + f"{CREDENTIAL_VAR} is set inside the DTU ({base_url}) but the server did not " + f"answer GET {base_url}/models from inside the container, so the vllm suite is " + f"skipped rather than failed: an unreachable endpoint is a fact about the " + f"operator's server, not a defect in amplifier-agent.\n" + f"Check that the vLLM server is running and bound to 0.0.0.0 (not 127.0.0.1), " + f"and that the host firewall permits traffic from the container bridge.\n" + f"Probe detail: {detail}" + ) + + +@pytest.fixture(scope="session") +def vllm_endpoint(dtu_id: str) -> dict[str, Any]: + """Skip unless a vLLM server is configured AND answering inside the DTU. + + Both checks run inside the container rather than on the host, because that is + where they matter: the host can have the variable exported and the DTU still not + have it if the container predates the export, and the host can reach + ``localhost:8007`` while the container cannot reach the gateway IP at all. + + Returns the resolved ``base_url`` and the model ids the server advertises. + """ + presence = dtu.exec_json(dtu_id, ["bash", "-lc", _PRESENCE_PROBE]) + if "endpoint=present" not in str(presence.get("stdout", "")): + pytest.skip(_SKIP_UNSET) + + # Read back the value the container actually holds -- the post-rewrite gateway + # URL, which is not what the host exported and is worth reporting in failures. + resolved = dtu.exec_json(dtu_id, ["bash", "-lc", f'printf "%s" "${CREDENTIAL_VAR}"']) + base_url = str(resolved.get("stdout", "")).strip() + + # One request that doubles as the reachability probe and the model discovery. + # --fail-with-body so a 4xx/5xx is a non-zero exit rather than a body we would + # then have to parse to notice the failure. + probe = dtu.exec_json( + dtu_id, + ["bash", "-lc", f"curl -sS --fail-with-body --max-time 15 {shlex.quote(base_url)}/models"], + ) + if probe.get("exit_code") != 0: + pytest.skip(_skip_unreachable(base_url, f"exit={probe.get('exit_code')} {probe.get('stderr', '')}".strip())) + + try: + payload = json.loads(str(probe.get("stdout", ""))) + except json.JSONDecodeError as exc: + pytest.skip(_skip_unreachable(base_url, f"/models did not return JSON: {exc}")) + + model_ids = [m["id"] for m in payload.get("data", []) if isinstance(m, dict) and m.get("id")] + if not model_ids: + pytest.skip(_skip_unreachable(base_url, f"/models returned no model ids: {payload!r}")) + + return {"base_url": base_url, "model_ids": model_ids} + + +@pytest.fixture(scope="session") +def vllm_model(dtu_id: str, vllm_endpoint: dict[str, Any]) -> str: + """The model id to pin in the host config. + + ``VLLM_MODEL`` wins when the operator set it. Otherwise the first id the server + advertises is used, which is the right default for a single-model vLLM process + (the common case -- vLLM serves one model per instance). + + Falling back to discovery rather than to the provider module's own default is + deliberate. The module defaults to a model id that has no reason to exist on an + arbitrary self-hosted server, so leaving ``default_model`` unset would turn a + perfectly good setup into a confusing 404 from the operator's own server. + """ + requested = str(dtu.exec_json(dtu_id, ["bash", "-lc", 'printf "%s" "$VLLM_MODEL"']).get("stdout", "")).strip() + if requested: + return requested + return str(vllm_endpoint["model_ids"][0]) + + +@pytest.fixture(scope="session") +def vllm_config(dtu_id: str, vllm_endpoint: dict[str, Any], vllm_model: str) -> str: + """Render the vllm host config, push it into the DTU, return its in-container path. + + The config is generated rather than a static fixture file because the model id is + not knowable ahead of time -- it is whatever the operator's server is serving. + + ``base_url`` is deliberately NOT written here. ``VLLM_BASE_URL`` already resolves + from the environment, and ``_reassert_protected_keys`` re-asserts env-resolved + credential fields on top of any ``provider.config`` overlay, so a ``base_url`` in + this file would be silently overwritten. Writing it would create the false + impression that the host config is what points at the server. + """ + config = { + "provider": {"module": "vllm", "config": {"default_model": vllm_model}}, + "approval": {"mode": "yes"}, + } + + with tempfile.TemporaryDirectory(prefix="aa-e2e-vllm-") as tmp: + local = Path(tmp) / "host-config-vllm.json" + local.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + dtu.exec_json(dtu_id, ["mkdir", "-p", DTU_DIR]) + dtu.push_file(dtu_id, str(local), CONFIG_PATH) + + return CONFIG_PATH diff --git a/tests/e2e/suites/vllm/test_vllm.py b/tests/e2e/suites/vllm/test_vllm.py new file mode 100644 index 00000000..7a478c32 --- /dev/null +++ b/tests/e2e/suites/vllm/test_vllm.py @@ -0,0 +1,43 @@ +"""End-to-end coverage for the ``vllm`` provider. + +Proves the four things amplifier-agent became responsible for when vllm was +registered as the ninth provider: the credential report knows it and resolves its +endpoint from the environment, its models list off that endpoint, a host config +naming it validates, and a session runs on it. + +The suite skips wholesale when VLLM_BASE_URL is absent from the DTU, and also when +it is present but the server does not answer. See ``conftest.py`` for why an +unreachable endpoint is a skip rather than a failure. +""" + +from __future__ import annotations + +import pytest +from framework import harness +from framework.harness import E2ECase + +from suites.vllm.cases import MODELS, SMOKE, WIRING + +pytestmark = pytest.mark.dtu + + +def _ids(cases: list[E2ECase]) -> list[str]: + return [c.name for c in cases] + + +@pytest.mark.parametrize("case", WIRING, ids=_ids(WIRING)) +def test_vllm_wiring(case: E2ECase, dtu_id: str, vllm_config: str) -> None: + """Catalog registration, endpoint resolution, and config validation.""" + harness.run_cli_case(dtu_id, case) + + +@pytest.mark.parametrize("case", MODELS, ids=_ids(MODELS)) +def test_vllm_models_list(case: E2ECase, dtu_id: str, vllm_config: str) -> None: + """A live model listing off the configured server comes back non-empty.""" + harness.run_cli_case(dtu_id, case) + + +@pytest.mark.parametrize("case", SMOKE, ids=_ids(SMOKE)) +def test_vllm_basic_reply(case: E2ECase, dtu_id: str, vllm_config: str) -> None: + """One real session completes end to end against the vLLM server.""" + harness.run_cli_case(dtu_id, case)