feat: support OpenAI model providers

README.md

@@ -135,6 +135,33 @@ uv run coding-agent-bench run \
     --server-url http://my.server.url
 ```
 
+OpenAI-compatible endpoints are the default provider and require `--server-url`.
+To use OpenAI directly, set `OPENAI_API_KEY` in your environment, select the `openai` provider, and omit `--server-url`:
+
+```sh
+export OPENAI_API_KEY=<your-openai-api-key>
+uv run coding-agent-bench run \
+    --agent codex \
+    --dataset scale-ai/swe-bench-pro \
+    --model-name gpt-5 \
+    --model-provider openai
+```
+
+Native OpenAI is supported by Codex, OpenClaw, OpenCode, and Pi.
+Claude Code does not support OpenAI, and Oracle does not use a model provider.
+The key must be visible to the process launching Harbor; it is intentionally not passed in Harbor's `--ae` arguments because those are serialized into job metadata.
+
+OpenRouter is also selected explicitly and does not use `--server-url`:
+
+```sh
+export OPENROUTER_API_KEY=<your-openrouter-api-key>
+uv run coding-agent-bench run \
+    --agent codex \
+    --dataset scale-ai/swe-bench-pro \
+    --model-name openai/gpt-5 \
+    --model-provider openrouter
+```
+
 If you want to see a preview of Harbor command that would be run for a given set of arguments without actually running the job, add the `--dry-run` flag.
 
 > [!note]
@@ -195,7 +222,7 @@ sequenceDiagram
     ```sh
     oc apply -f deploy/job-queue-service.yml
     ```
-6. (Optional) To run jobs against OpenRouter (`server_url: openrouter`), create
+6. (Optional) To run jobs against OpenRouter (`model_provider: openrouter`), create
    an `openrouter-api-key` secret. Job pods mount it automatically (it is
    optional, so non-OpenRouter jobs are unaffected):
     ```yaml
@@ -207,10 +234,23 @@ sequenceDiagram
       OPENROUTER_API_KEY: <your-openrouter-api-key>
     type: Opaque
     ```
-    The queue service itself also needs `OPENROUTER_API_KEY` in its environment
-    to validate OpenRouter jobs at request time. Add it to `job-queue-secret`
-    (which the service already loads) or `envFrom` the `openrouter-api-key`
-    secret in `deploy/job-queue-service.yml`.
+    `deploy/job-queue-service.yml` loads this secret into the queue service,
+    and OpenRouter job pods reference the same secret automatically.
+
+7. (Optional) To run jobs against OpenAI-provided models, create an
+   `openai-api-key` secret. Submit these jobs with `model_provider: openai`;
+   they do not use `server_url`:
+   ```yaml
+   apiVersion: v1
+   kind: Secret
+   metadata:
+     name: openai-api-key
+   stringData:
+     OPENAI_API_KEY: <your-openai-api-key>
+   type: Opaque
+   ```
+   `deploy/job-queue-service.yml` loads this secret into the queue service,
+   and OpenAI job pods reference the same secret automatically.
 
 Get the route for the deployed service:
 
@@ -233,6 +273,18 @@ Queue up a new benchmark task:
 curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "test", "agent": "pi", "dataset": "swe-bench/swe-bench-verified", "model_name": "qwen3.6-27b", "server_url": "<server-url>", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: <your-api-key>"
 ```
 
+For OpenRouter, set `model_provider` and omit `server_url`:
+
+```sh
+curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "openrouter-test", "agent": "codex", "dataset": "swe-bench/swe-bench-verified", "model_name": "openai/gpt-5", "model_provider": "openrouter", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: <your-api-key>"
+```
+
+For OpenAI, set `model_provider` and omit `server_url`:
+
+```sh
+curl -X POST $JOB_QUEUE_URL/jobs -d '{"job_name": "openai-test", "agent": "codex", "dataset": "swe-bench/swe-bench-verified", "model_name": "gpt-5", "model_provider": "openai", "n_tasks": 1}' -H "Content-Type: application/json" -H "X-API-Key: <your-api-key>"
+```
+
 ```json
 {
     "message":"Job created.",

deploy/job-queue-service.yml

@@ -58,6 +58,18 @@ spec:
         env:
         - name: JOB_STORE_PATH
           value: /app/data/jobs.db
+        - name: OPENAI_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: openai-api-key
+              key: OPENAI_API_KEY
+              optional: true
+        - name: OPENROUTER_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: openrouter-api-key
+              key: OPENROUTER_API_KEY
+              optional: true
         envFrom:
         - secretRef:
             name: job-queue-secret
@@ -109,4 +121,4 @@ spec:
     kind: Service
     name: job-queue-service
     weight: 100
-  wildcardPolicy: None
+  wildcardPolicy: None

src/coding_agent_bench/agents/__init__.py

@@ -1,4 +1,5 @@
 from coding_agent_bench.agents.base import AgentConfig
+from coding_agent_bench.providers import ModelProvider
 from coding_agent_bench.agents.configs import (
     ClaudeCodeAgentConfig,
     CodexAgentConfig,
@@ -28,3 +29,6 @@ def get_agent_config(name: str) -> AgentConfig:
             f"Unsupported agent type '{name}'. Choose from: {list(AGENT_REGISTRY.keys())}"
         )
     return config
+
+
+__all__ = ["AGENT_CONFIGS", "AGENT_REGISTRY", "ModelProvider", "get_agent_config"]

src/coding_agent_bench/agents/base.py

@@ -1,7 +1,13 @@
-from abc import ABC, abstractmethod
+from abc import ABC
 from dataclasses import dataclass
 from typing import Any
 
+from coding_agent_bench.providers import (
+    ModelProvider,
+    ProviderConfig,
+    resolve_provider,
+)
+
 
 @dataclass
 class AgentConfigResult:
@@ -17,8 +23,38 @@ class AgentConfig(ABC):
 
     name: str
     version: str | None = None
+    supported_model_providers = frozenset(ModelProvider)
+
+    def _configure_openai_compatible(
+        self, provider: ProviderConfig, **kwargs
+    ) -> AgentConfigResult:
+        raise NotImplementedError(
+            f"{self.name} does not support OpenAI-compatible endpoints"
+        )
+
+    def _configure_openai(self, provider: ProviderConfig, **kwargs) -> AgentConfigResult:
+        raise NotImplementedError(f"{self.name} does not support OpenAI")
+
+    def _configure_openrouter(
+        self, provider: ProviderConfig, **kwargs
+    ) -> AgentConfigResult:
+        return self._configure_openai_compatible(provider=provider, **kwargs)
 
-    @abstractmethod
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def configure(
+        self,
+        model_provider: ModelProvider = ModelProvider.OPENAI_COMPATIBLE,
+        **kwargs,
+    ) -> AgentConfigResult:
         """Return agent-specific model, env vars, and mounts. Receives all build() kwargs."""
-        ...
+        model_provider = ModelProvider(model_provider)
+        server_url = kwargs.pop("server_url", None)
+        if model_provider not in self.supported_model_providers:
+            raise ValueError(
+                f"{self.name} does not support {model_provider.value}"
+            )
+        provider = resolve_provider(server_url, model_provider)
+        if provider.model_provider == ModelProvider.OPENAI:
+            return self._configure_openai(provider=provider, **kwargs)
+        if provider.model_provider == ModelProvider.OPENROUTER:
+            return self._configure_openrouter(provider=provider, **kwargs)
+        return self._configure_openai_compatible(provider=provider, **kwargs)

src/coding_agent_bench/agents/configs.py

@@ -4,21 +4,21 @@
 
 from coding_agent_bench.agents.base import AgentConfig, AgentConfigResult
 from coding_agent_bench.helpers.codex import codex_create_toml
-from coding_agent_bench.providers import (
-    OPENROUTER_API_KEY_ENV,
-    is_openrouter,
-    resolve_provider,
-)
+from coding_agent_bench.providers import ModelProvider
 
 
 class OracleAgentConfig(AgentConfig):
     """Non-LLM oracle agent. Passes the model through with no extra configuration."""
 
     name = "oracle"
-
-    def configure(self, **kwargs) -> AgentConfigResult:
-        if is_openrouter(kwargs["server_url"]):
-            raise ValueError("oracle is a non-LLM agent and cannot use OpenRouter")
+    supported_model_providers = frozenset(
+        {ModelProvider.OPENAI_COMPATIBLE, ModelProvider.OPENAI}
+    )
+
+    def configure(self, model_provider=ModelProvider.OPENAI_COMPATIBLE, **kwargs) -> AgentConfigResult:
+        model_provider = ModelProvider(model_provider)
+        if model_provider not in self.supported_model_providers:
+            raise ValueError(f"oracle cannot use {model_provider.value}")
         return AgentConfigResult(model=kwargs["model_name"])
 
 
@@ -27,23 +27,24 @@ class ClaudeCodeAgentConfig(AgentConfig):
 
     name = "claude-code"
     version = "2.1.220"
+    supported_model_providers = frozenset(
+        {ModelProvider.OPENAI_COMPATIBLE, ModelProvider.OPENROUTER}
+    )
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
-        base_url, api_key = resolve_provider(server_url)
         agent_env = {
-            "ANTHROPIC_BASE_URL": base_url,
+            "ANTHROPIC_BASE_URL": provider.base_url,
             "ANTHROPIC_MODEL": model_name,
             "ANTHROPIC_DEFAULT_OPUS_MODEL": model_name,
             "ANTHROPIC_DEFAULT_SONNET_MODEL": model_name,
             "ANTHROPIC_DEFAULT_HAIKU_MODEL": model_name,
         }
-        if api_key:
+        if provider.api_key:
             # OpenRouter's Claude Code integration expects a blank
             # ANTHROPIC_API_KEY and the key in ANTHROPIC_AUTH_TOKEN.
             agent_env["ANTHROPIC_API_KEY"] = ""
-            agent_env["ANTHROPIC_AUTH_TOKEN"] = api_key
+            agent_env["ANTHROPIC_AUTH_TOKEN"] = provider.api_key
         else:
             agent_env["ANTHROPIC_API_KEY"] = "sk-no-key-required"
         return AgentConfigResult(model=model_name, agent_env=agent_env)
@@ -55,17 +56,15 @@ class CodexAgentConfig(AgentConfig):
     name = "codex"
     version = "0.145.0"
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
-        base_url, api_key = resolve_provider(server_url)
 
         outpath = Path("config.toml").absolute()
         codex_create_toml(
             model_name=model_name,
-            server_url=base_url,
+            server_url=provider.base_url,
             outpath=outpath,
-            openrouter=bool(api_key),
+            openrouter=provider.model_provider == ModelProvider.OPENROUTER,
         )
         print(f"Created config.toml at {outpath}")
 
@@ -78,51 +77,55 @@ def configure(self, **kwargs) -> AgentConfigResult:
         ]
 
         agent_env = {"CODEX_HOME": "/root/.codex/"}
-        if api_key:
-            agent_env[OPENROUTER_API_KEY_ENV] = api_key
+        if provider.api_key:
+            agent_env[provider.api_key_env] = provider.api_key
 
         return AgentConfigResult(
             model="vllm/" + model_name,
             agent_env=agent_env,
             mounts=mounts,
         )
 
+    def _configure_openai(self, provider, **kwargs) -> AgentConfigResult:
+        # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae.
+        return AgentConfigResult(model="openai/" + kwargs["model_name"])
+
 
 class OpenClawAgentConfig(AgentConfig):
     """OpenClaw agent. Configures OpenAI-compatible API env vars."""
 
     name = "openclaw"
     version = "2026.6.1"
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
-        base_url, api_key = resolve_provider(server_url)
         agent_env = {
-            "OPENAI_BASE_URL": base_url.rstrip("/").removesuffix("/v1") + "/v1",
-            "OPENAI_API_KEY": api_key or "sk-no-key-required",
+            "OPENAI_BASE_URL": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1",
+            "OPENAI_API_KEY": provider.api_key or "sk-no-key-required",
         }
         return AgentConfigResult(model="vllm/" + model_name, agent_env=agent_env)
 
+    def _configure_openai(self, provider, **kwargs) -> AgentConfigResult:
+        # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae.
+        return AgentConfigResult(model="openai/" + kwargs["model_name"])
+
 
 class OpenCodeAgentConfig(AgentConfig):
     """OpenCode agent. Builds a JSON config with vLLM provider and context/output limits."""
 
     name = "opencode"
     version = "1.18.1"
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
         model_max_len = kwargs.get("model_max_len", 262000)
-        base_url, api_key = resolve_provider(server_url)
 
         model = "vllm/" + model_name
         context_limit = int(model_max_len * 0.75)
         output_limit = int(model_max_len * 0.25)
-        options = {"baseURL": base_url.rstrip("/").removesuffix("/v1") + "/v1"}
-        if api_key:
-            options["apiKey"] = api_key
+        options = {"baseURL": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1"}
+        if provider.api_key:
+            options["apiKey"] = provider.api_key
         opencode_config = {
             "$schema": "https://opencode.ai/config.json",
             "model": model,
@@ -146,22 +149,26 @@ def configure(self, **kwargs) -> AgentConfigResult:
         }
         return AgentConfigResult(model=model, agent_env=agent_env)
 
+    def _configure_openai(self, provider, **kwargs) -> AgentConfigResult:
+        # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae.
+        return AgentConfigResult(model="openai/" + kwargs["model_name"])
+
 
 class OpenHandsSdkAgentConfig(AgentConfig):
     """OpenHands agent. Sets environment variables for a vLLM provider."""
 
     name = "openhands-sdk"
+    supported_model_providers = frozenset({ModelProvider.OPENAI_COMPATIBLE})
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
 
         # Set LLM API in host environment
         os.environ["LLM_API_KEY"] = "NONE"
 
         # Configure the environment
         model = "hosted_vllm/" + model_name
-        api_base = server_url.rstrip("/").removesuffix("/v1") + "/v1"
+        api_base = provider.base_url.rstrip("/").removesuffix("/v1") + "/v1"
 
         agent_env = {
             "HOSTED_VLLM_API_BASE": api_base,
@@ -175,18 +182,16 @@ class PiAgentConfig(AgentConfig):
     name = "pi"
     version = "0.73.1"
 
-    def configure(self, **kwargs) -> AgentConfigResult:
+    def _configure_openai_compatible(self, provider, **kwargs) -> AgentConfigResult:
         model_name = kwargs["model_name"]
-        server_url = kwargs["server_url"]
         model_max_len = kwargs.get("model_max_len", 262000)
-        base_url, api_key = resolve_provider(server_url)
 
         models_json = {
             "providers": {
                 "vllm": {
-                    "baseUrl": base_url.rstrip("/").removesuffix("/v1") + "/v1",
+                    "baseUrl": provider.base_url.rstrip("/").removesuffix("/v1") + "/v1",
                     "api": "openai-completions",
-                    "apiKey": api_key or "NONE",
+                    "apiKey": provider.api_key or "NONE",
                     "models": [
                         {
                             "id": model_name,
@@ -213,3 +218,7 @@ def configure(self, **kwargs) -> AgentConfigResult:
 
         agent_env = {"PI_OFFLINE": "1", "PI_CODING_AGENT_DIR": "/root/.pi/agent"}
         return AgentConfigResult(model="vllm/" + model_name, agent_env=agent_env, mounts=mounts)
+
+    def _configure_openai(self, provider, **kwargs) -> AgentConfigResult:
+        # Harbor inherits OPENAI_API_KEY; agent_env would serialize it via --ae.
+        return AgentConfigResult(model="openai/" + kwargs["model_name"])