From f6e432e5651b9028cc6d2b69c32b67f5af4f6ce5 Mon Sep 17 00:00:00 2001 From: Taylor Agarwal Date: Wed, 2 Sep 2026 11:51:35 -0400 Subject: [PATCH] feat: support OpenAI model providers --- README.md | 62 +++++++++++- deploy/job-queue-service.yml | 14 ++- src/coding_agent_bench/agents/__init__.py | 4 + src/coding_agent_bench/agents/base.py | 44 ++++++++- src/coding_agent_bench/agents/configs.py | 91 ++++++++++-------- src/coding_agent_bench/api.py | 111 ++++++++++++++++------ src/coding_agent_bench/builder.py | 23 ++++- src/coding_agent_bench/cli.py | 19 +++- src/coding_agent_bench/job.py | 63 +++++++----- src/coding_agent_bench/providers.py | 87 +++++++++++++---- src/coding_agent_bench/ui.py | 25 +++-- tests/openrouter/test_agent_configs.py | 53 +++++++++-- tests/openrouter/test_api_validation.py | 48 +++++++--- tests/openrouter/test_codex.py | 5 +- tests/openrouter/test_job_spec.py | 38 ++++++-- tests/openrouter/test_openai.py | 45 +++++++++ tests/openrouter/test_providers.py | 51 ++++++---- 17 files changed, 604 insertions(+), 179 deletions(-) create mode 100644 tests/openrouter/test_openai.py diff --git a/README.md b/README.md index 697a05c..c61af4d 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,33 @@ uv run coding-agent-bench run \ --server-url http://my.server.url ``` +OpenAI-compatible endpoints are the default provider and require `--server-url`. +To use OpenAI directly, set `OPENAI_API_KEY` in your environment, select the `openai` provider, and omit `--server-url`: + +```sh +export OPENAI_API_KEY= +uv run coding-agent-bench run \ + --agent codex \ + --dataset scale-ai/swe-bench-pro \ + --model-name gpt-5 \ + --model-provider openai +``` + +Native OpenAI is supported by Codex, OpenClaw, OpenCode, and Pi. +Claude Code does not support OpenAI, and Oracle does not use a model provider. +The key must be visible to the process launching Harbor; it is intentionally not passed in Harbor's `--ae` arguments because those are serialized into job metadata. + +OpenRouter is also selected explicitly and does not use `--server-url`: + +```sh +export OPENROUTER_API_KEY= +uv run coding-agent-bench run \ + --agent codex \ + --dataset scale-ai/swe-bench-pro \ + --model-name openai/gpt-5 \ + --model-provider openrouter +``` + If you want to see a preview of Harbor command that would be run for a given set of arguments without actually running the job, add the `--dry-run` flag. > [!note] @@ -195,7 +222,7 @@ sequenceDiagram ```sh oc apply -f deploy/job-queue-service.yml ``` -6. (Optional) To run jobs against OpenRouter (`server_url: openrouter`), create +6. (Optional) To run jobs against OpenRouter (`model_provider: openrouter`), create an `openrouter-api-key` secret. Job pods mount it automatically (it is optional, so non-OpenRouter jobs are unaffected): ```yaml @@ -207,10 +234,23 @@ sequenceDiagram OPENROUTER_API_KEY: type: Opaque ``` - The queue service itself also needs `OPENROUTER_API_KEY` in its environment - to validate OpenRouter jobs at request time. Add it to `job-queue-secret` - (which the service already loads) or `envFrom` the `openrouter-api-key` - secret in `deploy/job-queue-service.yml`. + `deploy/job-queue-service.yml` loads this secret into the queue service, + and OpenRouter job pods reference the same secret automatically. + +7. (Optional) To run jobs against OpenAI-provided models, create an + `openai-api-key` secret. Submit these jobs with `model_provider: openai`; + they do not use `server_url`: + ```yaml + apiVersion: v1 + kind: Secret + metadata: + name: openai-api-key + stringData: + OPENAI_API_KEY: + type: Opaque + ``` + `deploy/job-queue-service.yml` loads this secret into the queue service, + and OpenAI job pods reference the same secret automatically. Get the route for the deployed service: @@ -233,6 +273,18 @@ Queue up a new benchmark task: curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "test", "agent": "pi", "dataset": "swe-bench/swe-bench-verified", "model_name": "qwen3.6-27b", "server_url": "", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: " ``` +For OpenRouter, set `model_provider` and omit `server_url`: + +```sh +curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "openrouter-test", "agent": "codex", "dataset": "swe-bench/swe-bench-verified", "model_name": "openai/gpt-5", "model_provider": "openrouter", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: " +``` + +For OpenAI, set `model_provider` and omit `server_url`: + +```sh +curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "openai-test", "agent": "codex", "dataset": "swe-bench/swe-bench-verified", "model_name": "gpt-5", "model_provider": "openai", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: " +``` + ```json { "message":"Job created.", diff --git a/deploy/job-queue-service.yml b/deploy/job-queue-service.yml index d1cf16e..7024119 100644 --- a/deploy/job-queue-service.yml +++ b/deploy/job-queue-service.yml @@ -58,6 +58,18 @@ spec: env: - name: JOB_STORE_PATH value: /app/data/jobs.db + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: openai-api-key + key: OPENAI_API_KEY + optional: true + - name: OPENROUTER_API_KEY + valueFrom: + secretKeyRef: + name: openrouter-api-key + key: OPENROUTER_API_KEY + optional: true envFrom: - secretRef: name: job-queue-secret @@ -109,4 +121,4 @@ spec: kind: Service name: job-queue-service weight: 100 - wildcardPolicy: None \ No newline at end of file + wildcardPolicy: None diff --git a/src/coding_agent_bench/agents/__init__.py b/src/coding_agent_bench/agents/__init__.py index 7c2baa4..dd4431d 100644 --- a/src/coding_agent_bench/agents/__init__.py +++ b/src/coding_agent_bench/agents/__init__.py @@ -1,4 +1,5 @@ from coding_agent_bench.agents.base import AgentConfig +from coding_agent_bench.providers import ModelProvider from coding_agent_bench.agents.configs import ( ClaudeCodeAgentConfig, CodexAgentConfig, @@ -28,3 +29,6 @@ def get_agent_config(name: str) -> AgentConfig: f"Unsupported agent type '{name}'. Choose from: {list(AGENT_REGISTRY.keys())}" ) return config + + +__all__ = ["AGENT_CONFIGS", "AGENT_REGISTRY", "ModelProvider", "get_agent_config"] diff --git a/src/coding_agent_bench/agents/base.py b/src/coding_agent_bench/agents/base.py index 9d1dac2..336a064 100644 --- a/src/coding_agent_bench/agents/base.py +++ b/src/coding_agent_bench/agents/base.py @@ -1,7 +1,13 @@ -from abc import ABC, abstractmethod +from abc import ABC from dataclasses import dataclass from typing import Any +from coding_agent_bench.providers import ( + ModelProvider, + ProviderConfig, + resolve_provider, +) + @dataclass class AgentConfigResult: @@ -17,8 +23,38 @@ class AgentConfig(ABC): name: str version: str | None = None + supported_model_providers = frozenset(ModelProvider) + + def _configure_openai_compatible( + self, provider: ProviderConfig, **kwargs + ) -> AgentConfigResult: + raise NotImplementedError( + f"{self.name} does not support OpenAI-compatible endpoints" + ) + + def _configure_openai(self, provider: ProviderConfig, **kwargs) -> AgentConfigResult: + raise NotImplementedError(f"{self.name} does not support OpenAI") + + def _configure_openrouter( + self, provider: ProviderConfig, **kwargs + ) -> AgentConfigResult: + return self._configure_openai_compatible(provider=provider, **kwargs) - @abstractmethod - def configure(self, **kwargs) -> AgentConfigResult: + def configure( + self, + model_provider: ModelProvider = ModelProvider.OPENAI_COMPATIBLE, + **kwargs, + ) -> AgentConfigResult: """Return agent-specific model, env vars, and mounts. Receives all build() kwargs.""" - ... + model_provider = ModelProvider(model_provider) + server_url = kwargs.pop("server_url", None) + if model_provider not in self.supported_model_providers: + raise ValueError( + f"{self.name} does not support {model_provider.value}" + ) + provider = resolve_provider(server_url, model_provider) + if provider.model_provider == ModelProvider.OPENAI: + return self._configure_openai(provider=provider, **kwargs) + if provider.model_provider == ModelProvider.OPENROUTER: + return self._configure_openrouter(provider=provider, **kwargs) + return self._configure_openai_compatible(provider=provider, **kwargs) diff --git a/src/coding_agent_bench/agents/configs.py b/src/coding_agent_bench/agents/configs.py index 814863e..80f7337 100644 --- a/src/coding_agent_bench/agents/configs.py +++ b/src/coding_agent_bench/agents/configs.py @@ -4,21 +4,21 @@ from coding_agent_bench.agents.base import AgentConfig, AgentConfigResult from coding_agent_bench.helpers.codex import codex_create_toml -from coding_agent_bench.providers import ( - OPENROUTER_API_KEY_ENV, - is_openrouter, - resolve_provider, -) +from coding_agent_bench.providers import ModelProvider class OracleAgentConfig(AgentConfig): """Non-LLM oracle agent. Passes the model through with no extra configuration.""" name = "oracle" - - def configure(self, **kwargs) -> AgentConfigResult: - if is_openrouter(kwargs["server_url"]): - raise ValueError("oracle is a non-LLM agent and cannot use OpenRouter") + supported_model_providers = frozenset( + {ModelProvider.OPENAI_COMPATIBLE, ModelProvider.OPENAI} + ) + + def configure(self, model_provider=ModelProvider.OPENAI_COMPATIBLE, **kwargs) -> AgentConfigResult: + model_provider = ModelProvider(model_provider) + if model_provider not in self.supported_model_providers: + raise ValueError(f"oracle cannot use {model_provider.value}") return AgentConfigResult(model=kwargs["model_name"]) @@ -27,23 +27,24 @@ class ClaudeCodeAgentConfig(AgentConfig): name = "claude-code" version = "2.1.220" + supported_model_providers = frozenset( + {ModelProvider.OPENAI_COMPATIBLE, ModelProvider.OPENROUTER} + ) - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] - base_url, api_key = resolve_provider(server_url) agent_env = { - "ANTHROPIC_BASE_URL": base_url, + "ANTHROPIC_BASE_URL": provider.base_url, "ANTHROPIC_MODEL": model_name, "ANTHROPIC_DEFAULT_OPUS_MODEL": model_name, "ANTHROPIC_DEFAULT_SONNET_MODEL": model_name, "ANTHROPIC_DEFAULT_HAIKU_MODEL": model_name, } - if api_key: + if provider.api_key: # OpenRouter's Claude Code integration expects a blank # ANTHROPIC_API_KEY and the key in ANTHROPIC_AUTH_TOKEN. agent_env["ANTHROPIC_API_KEY"] = "" - agent_env["ANTHROPIC_AUTH_TOKEN"] = api_key + agent_env["ANTHROPIC_AUTH_TOKEN"] = provider.api_key else: agent_env["ANTHROPIC_API_KEY"] = "sk-no-key-required" return AgentConfigResult(model=model_name, agent_env=agent_env) @@ -55,17 +56,15 @@ class CodexAgentConfig(AgentConfig): name = "codex" version = "0.145.0" - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] - base_url, api_key = resolve_provider(server_url) outpath = Path("config.toml").absolute() codex_create_toml( model_name=model_name, - server_url=base_url, + server_url=provider.base_url, outpath=outpath, - openrouter=bool(api_key), + openrouter=provider.model_provider == ModelProvider.OPENROUTER, ) print(f"Created config.toml at {outpath}") @@ -78,8 +77,8 @@ def configure(self, **kwargs) -> AgentConfigResult: ] agent_env = {"CODEX_HOME": "/root/.codex/"} - if api_key: - agent_env[OPENROUTER_API_KEY_ENV] = api_key + if provider.api_key: + agent_env[provider.api_key_env] = provider.api_key return AgentConfigResult( model="vllm/" + model_name, @@ -87,6 +86,10 @@ def configure(self, **kwargs) -> AgentConfigResult: mounts=mounts, ) + def _configure_openai(self, provider, **kwargs) -> AgentConfigResult: + # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae. + return AgentConfigResult(model="openai/" + kwargs["model_name"]) + class OpenClawAgentConfig(AgentConfig): """OpenClaw agent. Configures OpenAI-compatible API env vars.""" @@ -94,16 +97,18 @@ class OpenClawAgentConfig(AgentConfig): name = "openclaw" version = "2026.6.1" - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] - base_url, api_key = resolve_provider(server_url) agent_env = { - "OPENAI_BASE_URL": base_url.rstrip("/").removesuffix("/v1") + "/v1", - "OPENAI_API_KEY": api_key or "sk-no-key-required", + "OPENAI_BASE_URL": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1", + "OPENAI_API_KEY": provider.api_key or "sk-no-key-required", } return AgentConfigResult(model="vllm/" + model_name, agent_env=agent_env) + def _configure_openai(self, provider, **kwargs) -> AgentConfigResult: + # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae. + return AgentConfigResult(model="openai/" + kwargs["model_name"]) + class OpenCodeAgentConfig(AgentConfig): """OpenCode agent. Builds a JSON config with vLLM provider and context/output limits.""" @@ -111,18 +116,16 @@ class OpenCodeAgentConfig(AgentConfig): name = "opencode" version = "1.18.1" - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] model_max_len = kwargs.get("model_max_len", 262000) - base_url, api_key = resolve_provider(server_url) model = "vllm/" + model_name context_limit = int(model_max_len * 0.75) output_limit = int(model_max_len * 0.25) - options = {"baseURL": base_url.rstrip("/").removesuffix("/v1") + "/v1"} - if api_key: - options["apiKey"] = api_key + options = {"baseURL": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1"} + if provider.api_key: + options["apiKey"] = provider.api_key opencode_config = { "$schema": "https://opencode.ai/config.json", "model": model, @@ -146,22 +149,26 @@ def configure(self, **kwargs) -> AgentConfigResult: } return AgentConfigResult(model=model, agent_env=agent_env) + def _configure_openai(self, provider, **kwargs) -> AgentConfigResult: + # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae. + return AgentConfigResult(model="openai/" + kwargs["model_name"]) + class OpenHandsSdkAgentConfig(AgentConfig): """OpenHands agent. Sets environment variables for a vLLM provider.""" name = "openhands-sdk" + supported_model_providers = frozenset({ModelProvider.OPENAI_COMPATIBLE}) - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] # Set LLM API in host environment os.environ["LLM_API_KEY"] = "NONE" # Configure the environment model = "hosted_vllm/" + model_name - api_base = server_url.rstrip("/").removesuffix("/v1") + "/v1" + api_base = provider.base_url.rstrip("/").removesuffix("/v1") + "/v1" agent_env = { "HOSTED_VLLM_API_BASE": api_base, @@ -175,18 +182,16 @@ class PiAgentConfig(AgentConfig): name = "pi" version = "0.73.1" - def configure(self, **kwargs) -> AgentConfigResult: + def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult: model_name = kwargs["model_name"] - server_url = kwargs["server_url"] model_max_len = kwargs.get("model_max_len", 262000) - base_url, api_key = resolve_provider(server_url) models_json = { "providers": { "vllm": { - "baseUrl": base_url.rstrip("/").removesuffix("/v1") + "/v1", + "baseUrl": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1", "api": "openai-completions", - "apiKey": api_key or "NONE", + "apiKey": provider.api_key or "NONE", "models": [ { "id": model_name, @@ -213,3 +218,7 @@ def configure(self, **kwargs) -> AgentConfigResult: agent_env = {"PI_OFFLINE": "1", "PI_CODING_AGENT_DIR": "/root/.pi/agent"} return AgentConfigResult(model="vllm/" + model_name, agent_env=agent_env, mounts=mounts) + + def _configure_openai(self, provider, **kwargs) -> AgentConfigResult: + # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae. + return AgentConfigResult(model="openai/" + kwargs["model_name"]) diff --git a/src/coding_agent_bench/api.py b/src/coding_agent_bench/api.py index f442385..25722d4 100644 --- a/src/coding_agent_bench/api.py +++ b/src/coding_agent_bench/api.py @@ -14,10 +14,13 @@ from pathlib import Path from coding_agent_bench.builder import SupportedAgent, HarborCommandBuilder -from coding_agent_bench.job import OpenshiftJob +from coding_agent_bench.job import JobOptions, OpenshiftJob from coding_agent_bench.nebius_utils import NebiusInstanceManager, RESOURCE_CONFIG_REGISTRY from coding_agent_bench.models import ModelConfig, MODEL_REGISTRY -from coding_agent_bench.providers import is_openrouter, resolve_provider, OPENROUTER_UNSUPPORTED_AGENTS +from coding_agent_bench.providers import ( + ModelProvider, + resolve_provider, +) from coding_agent_bench.agents import AGENT_REGISTRY from coding_agent_bench.ui import build_submit_form_html from coding_agent_bench import VERSION @@ -43,6 +46,7 @@ class QueuedJob(NamedTuple): command: list[str] server_url: str model_name: str + model_provider: ModelProvider _job_queue: list[QueuedJob] = [] _job_event = asyncio.Event() @@ -53,9 +57,9 @@ class QueuedJob(NamedTuple): NEBIUS_PREFIX = "nebius-" -def _parse_nebius_url(server_url: str) -> str | None: +def _parse_nebius_url(server_url: str | None) -> str | None: """Return the resource config name if server_url is a nebius placeholder, else None.""" - if server_url.startswith(NEBIUS_PREFIX): + if server_url and server_url.startswith(NEBIUS_PREFIX): return server_url[len(NEBIUS_PREFIX):] return None @@ -238,7 +242,10 @@ class CreateJobRequest(BaseModel): agent: SupportedAgent = Field(..., description="Agent to use") dataset: str = Field(..., description="Dataset name or path") model_name: str = Field(..., description="Model name") - server_url: str = Field(..., description="Model server URL; 'nebius-' (e.g. nebius-h200) for managed Nebius instances; or 'openrouter' to use OpenRouter (requires OPENROUTER_API_KEY on the server)") + model_provider: ModelProvider = Field( + ModelProvider.OPENAI_COMPATIBLE, description="Model API provider" + ) + server_url: Optional[str] = Field(None, description="Model server URL for OpenAI-compatible endpoints, or 'nebius-' (e.g. nebius-h200) for managed Nebius instances") dataset_pattern: Optional[str] = Field(None, description="Pattern to filter dataset tasks") n_concurrent: int = Field(1, description="Number of concurrent tasks") n_tasks: Optional[int] = Field(None, description="Total number of tasks to run") @@ -270,6 +277,7 @@ class JobResponse(BaseModel): agent: str dataset: str model_name: str + model_provider: ModelProvider server_url: str command: str status: JobStatus @@ -299,6 +307,7 @@ def _init_db(self): agent TEXT NOT NULL, dataset TEXT NOT NULL, model_name TEXT NOT NULL, + model_provider TEXT NOT NULL DEFAULT 'openai-compatible', server_url TEXT NOT NULL DEFAULT '', command TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'queued', @@ -309,15 +318,20 @@ def _init_db(self): columns = {row[1] for row in conn.execute("PRAGMA table_info(jobs)").fetchall()} if "server_url" not in columns: conn.execute("ALTER TABLE jobs ADD COLUMN server_url TEXT NOT NULL DEFAULT ''") + if "model_provider" not in columns: + conn.execute( + "ALTER TABLE jobs ADD COLUMN model_provider TEXT NOT NULL " + "DEFAULT 'openai-compatible'" + ) conn.commit() conn.close() - def insert(self, job_id: str, job_name: str, agent: str, dataset: str, model_name: str, server_url: str, command: list[str]): + def insert(self, job_id: str, job_name: str, agent: str, dataset: str, model_name: str, model_provider: ModelProvider, server_url: str, command: list[str]): """Add a new job to the tracking table.""" conn = self._connect() conn.execute( - "INSERT INTO jobs (job_id, job_name, agent, dataset, model_name, server_url, command, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - (job_id, job_name, agent, dataset, model_name, server_url, json.dumps(command), JobStatus.QUEUED.value), + "INSERT INTO jobs (job_id, job_name, agent, dataset, model_name, model_provider, server_url, command, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (job_id, job_name, agent, dataset, model_name, model_provider.value, server_url, json.dumps(command), JobStatus.QUEUED.value), ) conn.commit() conn.close() @@ -487,7 +501,7 @@ async def _best_effort_cleanup(oj: OpenshiftJob, signal: bool = False) -> str | return "; ".join(errors) if errors else None -async def _run_job(job_id: str, command: list[str], openrouter: bool = False): +async def _run_job(job_id: str, command: list[str], options: JobOptions = JobOptions()): """Run and monitor an Openshift Job.""" global _active_job @@ -499,9 +513,9 @@ async def _run_job(job_id: str, command: list[str], openrouter: bool = False): try: is_resume = len(command) == 3 and command[0] == "sh" and command[1] == "-c" if is_resume: - job_spec = oj._resume_job_spec(command[2]) + job_spec = oj._resume_job_spec(command[2], options=options) else: - job_spec = oj._job_spec(command, openrouter=openrouter) + job_spec = oj._job_spec(command, options=options) await oj._run_oc_command( ["apply", "-f", "-"], stdin_data=json.dumps(job_spec).encode(), @@ -609,7 +623,7 @@ async def _worker(): _job_event.clear() while _job_queue: _reorder_queue_for_nebius() - job_id, command, server_url, model_name = _job_queue.pop(0) + job_id, command, server_url, model_name, model_provider = _job_queue.pop(0) row = job_store.get(job_id) if not row or row["status"] != JobStatus.QUEUED.value: continue @@ -643,7 +657,11 @@ async def _worker(): if "--model-max-len" not in command and model_config is not None: command += ["--model-max-len", str(model_config.model_max_len)] - await _run_job(job_id, command, openrouter=is_openrouter(server_url)) + await _run_job( + job_id, + command, + options=JobOptions(model_provider=model_provider), + ) if nebius_instance_name and _nebius: await _nebius.mark_job_completed(nebius_instance_name) @@ -660,7 +678,7 @@ async def ui(): Intentionally left accessible to unauthenticated users as it does not expose any secret information or allow users to modify any job. """ - columns = ["job_id", "job_name", "agent", "dataset", "model_name", "server_url", "status", "error"] + columns = ["job_id", "job_name", "agent", "dataset", "model_name", "model_provider", "server_url", "status", "error"] def build_table(title: str, jobs: list[dict]) -> str: header = "".join(f"{col}" for col in columns) @@ -787,9 +805,11 @@ def build_cli_command(req: CreateJobRequest): "--agent", req.agent, "--dataset", req.dataset, "--model-name", req.model_name, - "--server-url", req.server_url, + "--model-provider", req.model_provider.value, "--environment", "openshift", ] + if req.server_url: + command += ["--server-url", req.server_url] # Add optional parameters if req.dataset_pattern: @@ -815,9 +835,21 @@ def build_cli_command(req: CreateJobRequest): @router.post("/jobs", response_model=CreateJobResponse) async def create_job(req: CreateJobRequest): """Create a new benchmark job.""" + agent_config = AGENT_REGISTRY[req.agent.value] + if req.model_provider not in agent_config.supported_model_providers: + raise HTTPException( + status_code=400, + detail=f"agent '{req.agent.value}' does not support {req.model_provider.value}", + ) + + if req.model_provider == ModelProvider.OPENAI and req.server_url: + raise HTTPException(status_code=400, detail="server_url does not apply to the OpenAI provider") + # Skip harbor command validation for nebius jobs (server_url is a placeholder) nebius_gpu_config = _parse_nebius_url(req.server_url) if nebius_gpu_config is not None: + if req.model_provider != ModelProvider.OPENAI_COMPATIBLE: + raise HTTPException(status_code=400, detail="Nebius model servers require the openai-compatible provider") if not _nebius: raise HTTPException(status_code=400, detail="Nebius is not enabled on this server") if nebius_gpu_config not in RESOURCE_CONFIG_REGISTRY: @@ -825,24 +857,29 @@ async def create_job(req: CreateJobRequest): status_code=400, detail=f"Unknown resource config '{nebius_gpu_config}'. Choose from: {', '.join(RESOURCE_CONFIG_REGISTRY)}", ) - elif is_openrouter(req.server_url): + elif req.model_provider == ModelProvider.OPENROUTER: # Skip HarborCommandBuilder().build() for openrouter jobs: build() runs # each agent's configure(), and PiAgentConfig.configure() writes the # real OpenRouter key to models.json on the API host's CWD as a side # effect. Validate cheaply instead, deferring dataset-existence checks # to run time (same trade-off as the nebius branch above). try: - resolve_provider(req.server_url) + resolve_provider(req.server_url, req.model_provider) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) - if req.agent.value in OPENROUTER_UNSUPPORTED_AGENTS: - raise HTTPException(status_code=400, detail=f"agent '{req.agent.value}' cannot use OpenRouter") + elif req.model_provider == ModelProvider.OPENAI: + if req.agent.value != "oracle": + try: + resolve_provider(req.server_url, req.model_provider) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) else: try: HarborCommandBuilder().build( agent=req.agent, dataset=req.dataset, model_name=req.model_name, + model_provider=req.model_provider, server_url=req.server_url, environment="openshift", dataset_pattern=req.dataset_pattern, @@ -859,8 +896,9 @@ async def create_job(req: CreateJobRequest): # Start the job job_id = str(uuid.uuid4()) - job_store.insert(job_id, req.job_name, req.agent.value, req.dataset, req.model_name, req.server_url, command) - _job_queue.append(QueuedJob(job_id, command, req.server_url, req.model_name)) + server_url = req.server_url or "" + job_store.insert(job_id, req.job_name, req.agent.value, req.dataset, req.model_name, req.model_provider, server_url, command) + _job_queue.append(QueuedJob(job_id, command, server_url, req.model_name, req.model_provider)) _job_event.set() # Return a success response @@ -974,10 +1012,28 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): original_server_url = job_row.get("server_url", "") effective_server_url = req.server_url or original_server_url + model_provider = ModelProvider(job_row["model_provider"]) + + if model_provider == ModelProvider.OPENAI: + if req.server_url: + raise HTTPException(status_code=400, detail="server_url does not apply to the OpenAI provider") + try: + resolve_provider(None, model_provider) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + if model_provider == ModelProvider.OPENROUTER: + if req.server_url: + raise HTTPException(status_code=400, detail="server_url does not apply to the OpenRouter provider") + try: + resolve_provider(None, model_provider) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) # Validate nebius URLs the same way create_job does nebius_gpu_config = _parse_nebius_url(effective_server_url) if nebius_gpu_config is not None: + if model_provider != ModelProvider.OPENAI_COMPATIBLE: + raise HTTPException(status_code=400, detail="Nebius model servers require the openai-compatible provider") if not _nebius: raise HTTPException(status_code=400, detail="Nebius is not enabled on this server") if nebius_gpu_config not in RESOURCE_CONFIG_REGISTRY: @@ -990,12 +1046,10 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): job_dir = f"/app/jobs/{shlex.quote(original_job_name)}" py_job_dir = f"/app/jobs/{original_job_name}" - # URL replacement only applies to real, changing hostnames (e.g. a new - # nebius instance IP). It is skipped for nebius placeholders (deferred to - # the worker) and for the openrouter sentinel, whose URL is static and - # already baked into the restored config. + # URL replacement only applies to changing OpenAI-compatible endpoints. + # Nebius placeholders are resolved later by the worker. url_replace_step = "" - if req.server_url and nebius_gpu_config is None and not is_openrouter(req.server_url): + if model_provider == ModelProvider.OPENAI_COMPATIBLE and req.server_url and nebius_gpu_config is None: url_replace_step = _build_url_replace_shell_step(req.server_url, py_job_dir) shell_command = ( @@ -1008,11 +1062,12 @@ async def resume_job(job_id: str, req: ResumeJobRequest = ResumeJobRequest()): ) command = ["sh", "-c", shell_command] + stored_server_url = effective_server_url or "" job_store.insert( resume_job_id, resume_job_name, job_row["agent"], - job_row["dataset"], job_row["model_name"], effective_server_url, command, + job_row["dataset"], job_row["model_name"], model_provider, stored_server_url, command, ) - _job_queue.append(QueuedJob(resume_job_id, command, effective_server_url, job_row["model_name"])) + _job_queue.append(QueuedJob(resume_job_id, command, stored_server_url, job_row["model_name"], model_provider)) _job_event.set() return { diff --git a/src/coding_agent_bench/builder.py b/src/coding_agent_bench/builder.py index 4c0149c..c363878 100644 --- a/src/coding_agent_bench/builder.py +++ b/src/coding_agent_bench/builder.py @@ -6,7 +6,7 @@ from harbor.models.environment_type import EnvironmentType -from coding_agent_bench.agents import get_agent_config +from coding_agent_bench.agents import ModelProvider, get_agent_config class SupportedAgent(str, Enum): @@ -113,8 +113,9 @@ def build( agent: str, dataset: str, model_name: str, - server_url: str, environment: Literal["docker", "openshift"], + model_provider: ModelProvider = ModelProvider.OPENAI_COMPATIBLE, + server_url: str | None = None, dataset_pattern: str = None, n_concurrent: int = 1, n_tasks: int = None, @@ -136,7 +137,25 @@ def build( raise ValueError(f"Invalid environment: {environment}") agent_config = get_agent_config(agent) + try: + model_provider = ModelProvider(model_provider) + except ValueError: + supported = ", ".join(provider.value for provider in ModelProvider) + raise ValueError( + f"Unsupported model provider '{model_provider}'. Choose from: {supported}" + ) from None + if agent_config.name != "oracle": + if model_provider == ModelProvider.OPENAI_COMPATIBLE and not server_url: + raise ValueError( + "server_url is required for OpenAI-compatible endpoints" + ) + if model_provider == ModelProvider.OPENAI and server_url: + raise ValueError("server_url does not apply to the OpenAI provider") + if model_provider == ModelProvider.OPENROUTER and server_url: + raise ValueError("server_url does not apply to the OpenRouter provider") + result = agent_config.configure( + model_provider=model_provider, model_name=model_name, server_url=server_url, model_max_len=model_max_len, diff --git a/src/coding_agent_bench/cli.py b/src/coding_agent_bench/cli.py index 075d2f3..3120c5b 100644 --- a/src/coding_agent_bench/cli.py +++ b/src/coding_agent_bench/cli.py @@ -7,9 +7,9 @@ import typer +from coding_agent_bench.agents import ModelProvider from coding_agent_bench.builder import HarborCommandBuilder, SupportedAgent -from coding_agent_bench.job import OpenshiftJob -from coding_agent_bench.providers import is_openrouter +from coding_agent_bench.job import JobOptions, OpenshiftJob from coding_agent_bench.manifest import deploy as deploy_model from coding_agent_bench.manifest import generate from coding_agent_bench.utils import cmd_to_string @@ -25,7 +25,13 @@ def run( ], dataset: Annotated[str, typer.Option(help="Dataset name or path")], model_name: Annotated[str, typer.Option(help="Model name")], - server_url: Annotated[str, typer.Option(help="Model server URL")], + model_provider: Annotated[ + ModelProvider, typer.Option(help="Model API provider") + ] = ModelProvider.OPENAI_COMPATIBLE, + server_url: Annotated[ + Optional[str], + typer.Option(help="Model server URL for OpenAI-compatible endpoints"), + ] = None, environment: Annotated[ str, typer.Option(help="Environment: docker or openshift") ] = "docker", @@ -86,7 +92,11 @@ def run( # Create the job try: - job.run(command, _before_script, openrouter=is_openrouter(server_url)) + job.run( + command, + _before_script, + options=JobOptions(model_provider=model_provider), + ) except KeyboardInterrupt: typer.echo("\nInterrupted — cleaning up remote job...") job.cleanup() @@ -99,6 +109,7 @@ def run( agent=agent, dataset=dataset, model_name=model_name, + model_provider=model_provider, server_url=server_url, environment=environment, dataset_pattern=dataset_pattern, diff --git a/src/coding_agent_bench/job.py b/src/coding_agent_bench/job.py index e995bd7..67f45af 100644 --- a/src/coding_agent_bench/job.py +++ b/src/coding_agent_bench/job.py @@ -3,6 +3,14 @@ import subprocess import asyncio import json +from dataclasses import dataclass + +from coding_agent_bench.providers import ModelProvider, PROVIDER_SECRETS + + +@dataclass(frozen=True) +class JobOptions: + model_provider: ModelProvider = ModelProvider.OPENAI_COMPATIBLE class OpenshiftJob: @@ -30,7 +38,28 @@ def __init__(self, job_name: str): self._job_name = job_name self._pod_name = f"coding-agent-bench--{self._job_name}"[:58] - def _resume_job_spec(self, shell_command: str) -> dict: + @staticmethod + def _provider_env(options: JobOptions) -> list[dict]: + secret = PROVIDER_SECRETS.get(options.model_provider) + if secret is None: + return [] + env_name, secret_name = secret + return [ + { + "name": env_name, + "valueFrom": { + "secretKeyRef": { + "name": secret_name, + "key": env_name, + "optional": True, + } + }, + } + ] + + def _resume_job_spec( + self, shell_command: str, options: JobOptions = JobOptions() + ) -> dict: """Build a pod spec for a resume job with a raw shell command.""" return { "apiVersion": "batch/v1", @@ -51,7 +80,7 @@ def _resume_job_spec(self, shell_command: str) -> dict: "args": [shell_command], "env": [ {"name": "HOME", "value": "/tmp"}, - ], + ] + self._provider_env(options), "volumeMounts": [{"name": "jobs", "mountPath": "/app/jobs"}], "envFrom": [ {"secretRef": {"name": "harbor-minio"}} @@ -67,24 +96,8 @@ def _job_spec( self, command: list[str], before_script: list[str] = None, - openrouter: bool = False, + options: JobOptions = JobOptions(), ) -> dict: - # Only openrouter jobs need the OpenRouter key, so scope the secret to - # them rather than exposing it to every job pod. - env: list[dict] = [] - if openrouter: - env.append( - { - "name": "OPENROUTER_API_KEY", - "valueFrom": { - "secretKeyRef": { - "name": "openrouter-api-key", - "key": "OPENROUTER_API_KEY", - "optional": True, - } - }, - } - ) return { "apiVersion": "batch/v1", "kind": "Job", @@ -109,7 +122,7 @@ def _job_spec( + " && mc mb --ignore-existing minio/results" + " && mc cp --recursive /app/jobs/ minio/results/" ], - "env": env, + "env": self._provider_env(options), "volumeMounts": [{"name": "jobs", "mountPath": "/app/jobs"}], "envFrom": [ {"secretRef": {"name": "harbor-minio"}} @@ -303,9 +316,9 @@ async def run_async( self, command: list[str], before_script: list[str] = None, - openrouter: bool = False, + options: JobOptions = JobOptions(), ): - job_spec = self._job_spec(command, before_script, openrouter=openrouter) + job_spec = self._job_spec(command, before_script, options=options) job_json = json.dumps(job_spec) try: @@ -327,11 +340,9 @@ def run( self, command: list[str], before_script: list[str] = None, - openrouter: bool = False, + options: JobOptions = JobOptions(), ): - return asyncio.run( - self.run_async(command, before_script, openrouter=openrouter) - ) + return asyncio.run(self.run_async(command, before_script, options=options)) async def _cleanup_async(self): await self._signal_job_pod() diff --git a/src/coding_agent_bench/providers.py b/src/coding_agent_bench/providers.py index 66fc518..e6de9e3 100644 --- a/src/coding_agent_bench/providers.py +++ b/src/coding_agent_bench/providers.py @@ -1,38 +1,83 @@ import os +from dataclasses import dataclass +from enum import Enum + + +class ModelProvider(str, Enum): + OPENAI = "openai" + OPENROUTER = "openrouter" + OPENAI_COMPATIBLE = "openai-compatible" -OPENROUTER_SENTINEL = "openrouter" # Base URL without the /v1 suffix. Agents append /v1 as needed (openclaw, # opencode, pi, codex); claude-code uses the base as-is for ANTHROPIC_BASE_URL. OPENROUTER_BASE_URL = "https://openrouter.ai/api" OPENROUTER_API_KEY_ENV = "OPENROUTER_API_KEY" +OPENAI_API_KEY_ENV = "OPENAI_API_KEY" +OPENAI_BASE_URL = "https://api.openai.com" -# Agents (SupportedAgent.value strings) that cannot use OpenRouter. This is -# only a fast, request-time 400 for create_job's lightweight openrouter -# validation branch; the authoritative guard lives in the agent's own -# configure() (see OracleAgentConfig), which still raises ValueError at -# build/run time regardless of this constant. -OPENROUTER_UNSUPPORTED_AGENTS = frozenset({"oracle"}) +@dataclass(frozen=True) +class ProviderConfig: + model_provider: ModelProvider + base_url: str | None + api_key: str | None + api_key_env: str | None -def is_openrouter(server_url: str) -> bool: - """Return True if server_url is the OpenRouter sentinel.""" - return server_url == OPENROUTER_SENTINEL +PROVIDER_SECRETS = { + ModelProvider.OPENAI: (OPENAI_API_KEY_ENV, "openai-api-key"), + ModelProvider.OPENROUTER: (OPENROUTER_API_KEY_ENV, "openrouter-api-key"), +} -def resolve_provider(server_url: str) -> tuple[str, str | None]: - """Resolve server_url to (base_url, api_key). +def resolve_provider( + server_url: str | None, + model_provider: ModelProvider = ModelProvider.OPENAI_COMPATIBLE, +) -> ProviderConfig: + """Resolve a model provider's endpoint and credentials. - For the "openrouter" sentinel, return the OpenRouter base URL and the key - from the OPENROUTER_API_KEY environment variable, raising ValueError if the - key is not set. For any other server_url, return (server_url, None) so - callers keep their existing no-auth behavior. + The OpenAI and OpenRouter services use their canonical URLs. Custom + OpenAI-compatible providers require an explicit server URL. """ - if is_openrouter(server_url): + model_provider = ModelProvider(model_provider) + if model_provider == ModelProvider.OPENAI: + if server_url: + raise ValueError("server_url does not apply to the OpenAI provider") + api_key = os.environ.get(OPENAI_API_KEY_ENV) + if not api_key: + raise ValueError( + f"{OPENAI_API_KEY_ENV} must be set when using the OpenAI provider" + ) + return ProviderConfig( + model_provider=model_provider, + base_url=OPENAI_BASE_URL, + api_key=api_key, + api_key_env=OPENAI_API_KEY_ENV, + ) + + if model_provider == ModelProvider.OPENROUTER: + if server_url: + raise ValueError("server_url does not apply to the OpenRouter provider") api_key = os.environ.get(OPENROUTER_API_KEY_ENV) if not api_key: raise ValueError( - f"server_url '{OPENROUTER_SENTINEL}' requires the " - f"{OPENROUTER_API_KEY_ENV} environment variable to be set" + f"{OPENROUTER_API_KEY_ENV} must be set when using the OpenRouter provider" ) - return OPENROUTER_BASE_URL, api_key - return server_url, None + return ProviderConfig( + model_provider=model_provider, + base_url=OPENROUTER_BASE_URL, + api_key=api_key, + api_key_env=OPENROUTER_API_KEY_ENV, + ) + + if not server_url: + raise ValueError("server_url is required for OpenAI-compatible endpoints") + if not server_url.startswith(("http://", "https://")): + raise ValueError( + "server_url must be an HTTP(S) URL for OpenAI-compatible endpoints" + ) + return ProviderConfig( + model_provider=model_provider, + base_url=server_url, + api_key=None, + api_key_env=None, + ) diff --git a/src/coding_agent_bench/ui.py b/src/coding_agent_bench/ui.py index 4553cbd..2d62942 100644 --- a/src/coding_agent_bench/ui.py +++ b/src/coding_agent_bench/ui.py @@ -94,13 +94,15 @@ def build_submit_form_html( const agent = document.getElementById('agent').value; const dataset = document.getElementById('dataset').value.trim(); const modelName = document.getElementById('model_name').value; + const modelProvider = document.getElementById('model_provider').value; const serverUrl = document.getElementById('server_url').value.trim(); if (!jobName) errors.push('Job name is required'); if (!agent) errors.push('Agent is required'); if (!dataset) errors.push('Dataset is required'); if (!modelName) errors.push('Model name is required'); - if (!serverUrl) errors.push('Server URL is required'); + if (modelProvider === 'openai-compatible' && !serverUrl) errors.push('Server URL is required'); + if (modelProvider !== 'openai-compatible' && serverUrl) errors.push('Server URL only applies to OpenAI-compatible providers'); // Validate server_url format if (serverUrl) {{ @@ -155,9 +157,11 @@ def build_submit_form_html( agent: document.getElementById('agent').value, dataset: document.getElementById('dataset').value.trim(), model_name: document.getElementById('model_name').value, - server_url: document.getElementById('server_url').value.trim(), + model_provider: document.getElementById('model_provider').value, n_concurrent: parseInt(document.getElementById('n_concurrent').value) || 1, }}; + const serverUrl = document.getElementById('server_url').value.trim(); + if (serverUrl) formData.server_url = serverUrl; // Advanced fields const datasetPattern = document.getElementById('dataset_pattern').value.trim(); @@ -257,14 +261,23 @@ def _build_basic_fields_html(models: list[str], agents: list[str], nebius_enable
- - {model_options} + + +
- - Server URL + {nebius_help} diff --git a/tests/openrouter/test_agent_configs.py b/tests/openrouter/test_agent_configs.py index 8c5912e..b3dc2ef 100644 --- a/tests/openrouter/test_agent_configs.py +++ b/tests/openrouter/test_agent_configs.py @@ -3,18 +3,22 @@ import pytest from coding_agent_bench.agents.configs import ( + CodexAgentConfig, ClaudeCodeAgentConfig, OpenClawAgentConfig, OpenCodeAgentConfig, OracleAgentConfig, PiAgentConfig, ) +from coding_agent_bench.providers import ModelProvider def test_openclaw_openrouter(monkeypatch): monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") result = OpenClawAgentConfig().configure( - model_name="openai/gpt-4o", server_url="openrouter" + model_provider=ModelProvider.OPENROUTER, + model_name="openai/gpt-4o", + server_url=None, ) assert result.agent_env["OPENAI_BASE_URL"] == "https://openrouter.ai/api/v1" assert result.agent_env["OPENAI_API_KEY"] == "sk-or-test" @@ -32,7 +36,11 @@ def test_openclaw_default_placeholder(monkeypatch): def test_pi_openrouter(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") - PiAgentConfig().configure(model_name="openai/gpt-4o", server_url="openrouter") + PiAgentConfig().configure( + model_provider=ModelProvider.OPENROUTER, + model_name="openai/gpt-4o", + server_url=None, + ) cfg = json.loads((tmp_path / "models.json").read_text()) provider = cfg["providers"]["vllm"] assert provider["baseUrl"] == "https://openrouter.ai/api/v1" @@ -50,7 +58,9 @@ def test_pi_default_placeholder(monkeypatch, tmp_path): def test_opencode_openrouter(monkeypatch): monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") result = OpenCodeAgentConfig().configure( - model_name="openai/gpt-4o", server_url="openrouter" + model_provider=ModelProvider.OPENROUTER, + model_name="openai/gpt-4o", + server_url=None, ) cfg = json.loads(result.agent_env["OPENCODE_CONFIG_CONTENT"]) provider = cfg["provider"]["vllm"] @@ -71,14 +81,20 @@ def test_opencode_default_no_key(monkeypatch): def test_oracle_rejects_openrouter(): - with pytest.raises(ValueError, match="OpenRouter"): - OracleAgentConfig().configure(model_name="m", server_url="openrouter") + with pytest.raises(ValueError, match="openrouter"): + OracleAgentConfig().configure( + model_provider=ModelProvider.OPENROUTER, + model_name="m", + server_url=None, + ) def test_claude_code_openrouter(monkeypatch): monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") result = ClaudeCodeAgentConfig().configure( - model_name="anthropic/claude-opus-5", server_url="openrouter" + model_provider=ModelProvider.OPENROUTER, + model_name="anthropic/claude-opus-5", + server_url=None, ) # OpenRouter's Claude Code integration: base URL without /v1, blank # ANTHROPIC_API_KEY, key in ANTHROPIC_AUTH_TOKEN. @@ -95,3 +111,28 @@ def test_claude_code_default_vllm(monkeypatch): assert result.agent_env["ANTHROPIC_BASE_URL"] == "http://vllm:8000" assert result.agent_env["ANTHROPIC_API_KEY"] == "sk-no-key-required" assert "ANTHROPIC_AUTH_TOKEN" not in result.agent_env + + +@pytest.mark.parametrize( + "config_class", [CodexAgentConfig, OpenClawAgentConfig, OpenCodeAgentConfig, PiAgentConfig] +) +def test_native_openai_uses_builtin_provider(monkeypatch, tmp_path, config_class): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-test") + result = config_class().configure( + model_provider=ModelProvider.OPENAI, + model_name="gpt-5", + server_url=None, + ) + assert result.model == "openai/gpt-5" + assert not result.agent_env + + +def test_claude_code_rejects_native_openai(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-test") + with pytest.raises(ValueError, match="does not support openai"): + ClaudeCodeAgentConfig().configure( + model_provider=ModelProvider.OPENAI, + model_name="gpt-5", + server_url=None, + ) diff --git a/tests/openrouter/test_api_validation.py b/tests/openrouter/test_api_validation.py index 299ee07..b773201 100644 --- a/tests/openrouter/test_api_validation.py +++ b/tests/openrouter/test_api_validation.py @@ -2,11 +2,10 @@ These exercise the full FastAPI app via TestClient rather than just the unit helpers in `providers.py` / `agents/configs.py`, to confirm `POST /jobs` -actually returns clean 400s for the two openrouter validation cases handled -in `create_job` (unsupported agent, missing OPENROUTER_API_KEY) instead of -falling through to HarborCommandBuilder().build() (which would write the real -key to disk for the `pi` agent — see providers.OPENROUTER_UNSUPPORTED_AGENTS -and api.create_job). +actually returns clean 400s for the two OpenRouter validation cases handled +in `create_job` (unsupported agent and missing OPENROUTER_API_KEY) instead of +falling through to HarborCommandBuilder().build(), which could write the real +key to a generated agent configuration. Env vars that gate app import-time and lifespan behavior (JOB_STORE_PATH, API_KEY) must be set BEFORE `coding_agent_bench.api` is imported, since @@ -27,11 +26,38 @@ import pytest from fastapi.testclient import TestClient # noqa: E402 -from coding_agent_bench.api import app # noqa: E402 +from coding_agent_bench.api import CreateJobRequest, app, build_cli_command # noqa: E402 +from coding_agent_bench.providers import ModelProvider # noqa: E402 HEADERS = {"X-API-Key": os.environ["API_KEY"]} +def test_openai_cli_command_uses_provider_without_server_url(): + request = CreateJobRequest( + job_name="openai-test", + agent="codex", + dataset="some-dataset", + model_name="gpt-5", + model_provider=ModelProvider.OPENAI, + ) + command = build_cli_command(request) + assert command[command.index("--model-provider") + 1] == "openai" + assert "--server-url" not in command + + +def test_openrouter_cli_command_uses_provider_without_server_url(): + request = CreateJobRequest( + job_name="openrouter-test", + agent="codex", + dataset="some-dataset", + model_name="openai/gpt-5", + model_provider=ModelProvider.OPENROUTER, + ) + command = build_cli_command(request) + assert command[command.index("--model-provider") + 1] == "openrouter" + assert "--server-url" not in command + + @pytest.fixture(scope="module") def client(): # Module-scoped: api.py keeps its job queue/worker state (including a @@ -54,7 +80,7 @@ def test_create_job_openrouter_unsupported_agent_returns_400(client, monkeypatch "agent": "oracle", "dataset": "some-dataset", "model_name": "openai/gpt-4o", - "server_url": "openrouter", + "model_provider": "openrouter", }, ) assert resp.status_code == 400, resp.text @@ -71,7 +97,7 @@ def test_create_job_openrouter_missing_key_returns_400(client, monkeypatch): "agent": "codex", "dataset": "some-dataset", "model_name": "openai/gpt-4o", - "server_url": "openrouter", + "model_provider": "openrouter", }, ) assert resp.status_code == 400, resp.text @@ -79,16 +105,14 @@ def test_create_job_openrouter_missing_key_returns_400(client, monkeypatch): def test_resume_openrouter_job_is_allowed(client, monkeypatch): - # Resuming an OpenRouter job works: the restored config already carries the - # OpenRouter URL and key, so no URL rewrite is needed (same as a static - # vLLM-route resume). + # Resuming an OpenRouter job restores its explicit provider and key. monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") from coding_agent_bench.api import JobStatus, job_store job_id = "resume-openrouter-test" job_store.insert( job_id, "or-job", "codex", "some-dataset", "openai/gpt-4o", - "openrouter", ["sh", "-c", "echo hi"], + ModelProvider.OPENROUTER, "", ["sh", "-c", "echo hi"], ) job_store.update_status(job_id, JobStatus.COMPLETED) diff --git a/tests/openrouter/test_codex.py b/tests/openrouter/test_codex.py index 21aa7f8..a1cc0a4 100644 --- a/tests/openrouter/test_codex.py +++ b/tests/openrouter/test_codex.py @@ -1,5 +1,6 @@ from coding_agent_bench.agents.configs import CodexAgentConfig from coding_agent_bench.helpers.codex import codex_create_toml +from coding_agent_bench.providers import ModelProvider def test_codex_toml_default_no_key(tmp_path): @@ -29,7 +30,9 @@ def test_codex_agent_openrouter(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") result = CodexAgentConfig().configure( - model_name="openai/gpt-4o", server_url="openrouter" + model_provider=ModelProvider.OPENROUTER, + model_name="openai/gpt-4o", + server_url=None, ) assert result.agent_env["OPENROUTER_API_KEY"] == "sk-or-test" content = (tmp_path / "config.toml").read_text() diff --git a/tests/openrouter/test_job_spec.py b/tests/openrouter/test_job_spec.py index 45a786b..cbb3da4 100644 --- a/tests/openrouter/test_job_spec.py +++ b/tests/openrouter/test_job_spec.py @@ -1,4 +1,5 @@ -from coding_agent_bench.job import OpenshiftJob +from coding_agent_bench.job import JobOptions, OpenshiftJob +from coding_agent_bench.providers import ModelProvider def _env_by_name(spec): @@ -8,7 +9,12 @@ def _env_by_name(spec): def test_job_spec_injects_openrouter_secret_only_when_openrouter(): job = OpenshiftJob(job_name="test") - env = _env_by_name(job._job_spec(["echo", "hi"], openrouter=True)) + env = _env_by_name( + job._job_spec( + ["echo", "hi"], + options=JobOptions(model_provider=ModelProvider.OPENROUTER), + ) + ) assert "OPENROUTER_API_KEY" in env ref = env["OPENROUTER_API_KEY"]["valueFrom"]["secretKeyRef"] assert ref["name"] == "openrouter-api-key" @@ -22,9 +28,29 @@ def test_job_spec_omits_openrouter_secret_for_non_openrouter(): assert "OPENROUTER_API_KEY" not in env -def test_resume_job_spec_never_injects_openrouter_secret(): - # Resuming an OpenRouter job is rejected at the API (400), so resume pods - # never need the key. +def test_job_spec_injects_openai_secret_only_when_enabled(): job = OpenshiftJob(job_name="test") - env = _env_by_name(job._resume_job_spec("echo hi")) + env = _env_by_name( + job._job_spec( + ["echo", "hi"], + options=JobOptions(model_provider=ModelProvider.OPENAI), + ) + ) + ref = env["OPENAI_API_KEY"]["valueFrom"]["secretKeyRef"] + assert ref == { + "name": "openai-api-key", + "key": "OPENAI_API_KEY", + "optional": True, + } assert "OPENROUTER_API_KEY" not in env + + +def test_resume_job_spec_injects_configured_provider_secret(): + job = OpenshiftJob(job_name="test") + env = _env_by_name( + job._resume_job_spec( + "echo hi", + options=JobOptions(model_provider=ModelProvider.OPENROUTER), + ) + ) + assert set(env) == {"HOME", "OPENROUTER_API_KEY"} diff --git a/tests/openrouter/test_openai.py b/tests/openrouter/test_openai.py new file mode 100644 index 0000000..db14600 --- /dev/null +++ b/tests/openrouter/test_openai.py @@ -0,0 +1,45 @@ +import pytest + +from coding_agent_bench.builder import HarborCommandBuilder +from coding_agent_bench.providers import ModelProvider + + +def test_builder_uses_native_openai_without_serializing_key(monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-test") + + command, _ = HarborCommandBuilder().build( + agent="codex", + dataset="example/dataset", + model_name="gpt-5", + model_provider=ModelProvider.OPENAI, + environment="docker", + ) + + model_index = command.index("--model") + assert command[model_index + 1] == "openai/gpt-5" + assert not any("OPENAI_API_KEY" in argument for argument in command) + assert not any("sk-openai-test" in argument for argument in command) + + +def test_builder_requires_openai_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + with pytest.raises(ValueError, match="OPENAI_API_KEY"): + HarborCommandBuilder().build( + agent="codex", + dataset="example/dataset", + model_name="gpt-5", + model_provider=ModelProvider.OPENAI, + environment="docker", + ) + + +def test_builder_requires_server_for_compatible_provider(): + with pytest.raises(ValueError, match="server_url is required"): + HarborCommandBuilder().build( + agent="codex", + dataset="example/dataset", + model_name="model", + model_provider=ModelProvider.OPENAI_COMPATIBLE, + environment="docker", + ) diff --git a/tests/openrouter/test_providers.py b/tests/openrouter/test_providers.py index 2edc87a..d43073e 100644 --- a/tests/openrouter/test_providers.py +++ b/tests/openrouter/test_providers.py @@ -1,20 +1,13 @@ import pytest from coding_agent_bench.providers import ( + ModelProvider, + OPENAI_BASE_URL, OPENROUTER_BASE_URL, - is_openrouter, resolve_provider, ) -def test_is_openrouter_true(): - assert is_openrouter("openrouter") is True - - -def test_is_openrouter_false(): - assert is_openrouter("https://vllm.example.com") is False - - def test_openrouter_base_url_excludes_v1(): # Agents append /v1 as needed; claude-code uses the base without it. assert OPENROUTER_BASE_URL == "https://openrouter.ai/api" @@ -23,19 +16,45 @@ def test_openrouter_base_url_excludes_v1(): def test_resolve_openrouter_returns_url_and_key(monkeypatch): monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") - base_url, api_key = resolve_provider("openrouter") - assert base_url == OPENROUTER_BASE_URL - assert api_key == "sk-or-test" + provider = resolve_provider(None, ModelProvider.OPENROUTER) + assert provider.base_url == OPENROUTER_BASE_URL + assert provider.api_key == "sk-or-test" + assert provider.model_provider == ModelProvider.OPENROUTER def test_resolve_openrouter_missing_key_raises(monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) with pytest.raises(ValueError, match="OPENROUTER_API_KEY"): - resolve_provider("openrouter") + resolve_provider(None, ModelProvider.OPENROUTER) def test_resolve_non_openrouter_passthrough(monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) - base_url, api_key = resolve_provider("https://vllm.example.com") - assert base_url == "https://vllm.example.com" - assert api_key is None + provider = resolve_provider("https://vllm.example.com") + assert provider.base_url == "https://vllm.example.com" + assert provider.api_key is None + + +def test_compatible_provider_rejects_non_url(): + with pytest.raises(ValueError, match="HTTP"): + resolve_provider("openrouter") + + +def test_resolve_openai_returns_native_provider(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-test") + provider = resolve_provider(None, ModelProvider.OPENAI) + assert provider.base_url == OPENAI_BASE_URL + assert provider.api_key == "sk-openai-test" + assert provider.api_key_env == "OPENAI_API_KEY" + + +def test_resolve_openai_requires_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + with pytest.raises(ValueError, match="OPENAI_API_KEY"): + resolve_provider(None, ModelProvider.OPENAI) + + +def test_resolve_openai_rejects_server_url(monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "sk-openai-test") + with pytest.raises(ValueError, match="server_url does not apply"): + resolve_provider("https://example.com", ModelProvider.OPENAI)