feat(evaluation): support OpenAI models in jobbench and deep-swe harnesses

.amplifier/evaluation/deep-swe/run.py

@@ -23,6 +23,7 @@
 sys.path.insert(0, str(Path(__file__).parent / "src"))
 
 from deepswe_agents import AGENTS, LOCAL_SOURCE_AGENTS
+from deepswe_agents.providers import API_KEY_VAR, provider_family
 
 DEEP_SWE_REPO = "https://github.com/datacurve-ai/deep-swe"
 DEEP_SWE_SHA = "435ee89ec2f2e2289f33b0da4f992f0b7b7266b9"
@@ -128,14 +129,21 @@ def list_task_names(tasks: Path) -> list[str]:
 # ----------------------------------------------------------------------
 
 
-def preflight(require_docker: bool = True) -> None:
+def preflight(model: str, require_docker: bool = True) -> None:
     pier = shutil.which("pier")
     if not pier:
         die(f"`pier` is not on PATH. Install it with:\n  {PIER_INSTALL_CMD}")
     check_pier_is_git_build(pier)
 
-    if not os.environ.get("ANTHROPIC_API_KEY"):
-        die("ANTHROPIC_API_KEY is not set. Export it before running.")
+    # Only the family actually under test is required. Demanding an Anthropic
+    # key for an OpenAI run would force a dummy value whose only effect is to
+    # satisfy this check -- a gate that has stopped gating anything.
+    # `--model` may carry a `<provider>/` prefix; the family is derived from the
+    # bare id, exactly as every adapter derives it.
+    bare_model = model.split("/", 1)[-1]
+    key_var = API_KEY_VAR[provider_family(bare_model)]
+    if not os.environ.get(key_var):
+        die(f"{key_var} is not set (required for model {model!r}). Export it before running.")
 
     if require_docker:
         if not shutil.which("docker"):
@@ -627,7 +635,7 @@ def main(argv: list[str]) -> int:
             die(f"--local-source path does not exist: {src}")
         args.local_source = str(src)
 
-    preflight(require_docker=not args.dry_run)
+    preflight(args.model, require_docker=not args.dry_run)
     tasks = ensure_tasks(tasks_dir)
 
     # Resolve ONCE, then hand the identical explicit list to every agent.

.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_agent.py

@@ -15,6 +15,7 @@
     AmplifierBaseAgent,
     _as_bool,
 )
+from deepswe_agents.providers import AGENT_PROVIDER_MODULE, OPENAI, REASONING_EFFORT
 
 HOST_CONFIG_PATH = "/root/host-config.json"
 
@@ -64,11 +65,29 @@ async def setup(self, environment: BaseEnvironment) -> None:
         )
 
     def _host_config(self) -> str:
+        # `provider.module` is what SELECTS the provider, not merely what
+        # configures it: single_turn reads it and hands the name to
+        # inject_provider, which mounts that module alone. The friendly names
+        # ("anthropic", "openai") are the ones its config merger accepts.
+        family = self.provider_family
+        provider_config: dict[str, Any] = {"default_model": self.model}
+        if family == OPENAI:
+            # provider-openai reads base_url from CONFIG ONLY. It does have an
+            # OPENAI_BASE_URL fallback, but only as an incidental behavior of
+            # AsyncOpenAI's own constructor -- not a contract this module reads.
+            # Stating it here is what makes the endpoint under benchmark
+            # explicit rather than an artifact of the SDK.
+            provider_config["base_url"] = self.base_url()
+            # Pin the reasoning effort rather than inheriting the model's own
+            # default, so this arm is benchmarked at the same effort as the
+            # other two. provider-openai validates the value at mount, so a
+            # typo fails the run loudly instead of silently reverting.
+            provider_config["reasoning_effort"] = REASONING_EFFORT
         config: dict[str, Any] = {
             "approval": {"mode": "yes"},
             "provider": {
-                "module": "anthropic",
-                "config": {"default_model": self.model},
+                "module": AGENT_PROVIDER_MODULE[family],
+                "config": provider_config,
             },
         }
         if self._raw_llm_payloads:

.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_foundation.py

@@ -11,11 +11,22 @@
 from pier.models.agent.network import NetworkAllowlist
 
 from deepswe_agents.base import UV_PRELUDE, WORKDIR, AmplifierBaseAgent
+from deepswe_agents.providers import (
+    ANTHROPIC,
+    API_KEY_VAR,
+    FOUNDATION_PROVIDER_MODULE,
+    FOUNDATION_PROVIDER_SOURCE,
+    OPENAI,
+    REASONING_EFFORT,
+)
 
 SETTINGS_PATH = "$HOME/.amplifier/settings.yaml"
 
 # Env vars used to smuggle provider config into the heredoc without it appearing
 # in argv. The key must never be logged; the base URL rides along for symmetry.
+# Deliberately provider-neutral names: the SOURCE var differs by family
+# (ANTHROPIC_API_KEY vs OPENAI_API_KEY) but the smuggling channel does not, so
+# the settings template has one shape regardless of which model is under test.
 API_KEY_ENV = "AMPLIFIER_BENCH_API_KEY"
 BASE_URL_ENV = "AMPLIFIER_BENCH_BASE_URL"
 
@@ -118,48 +129,79 @@ async def setup(self, environment: BaseEnvironment) -> None:
         await super().setup(environment)
         await self._write_settings(environment)
 
+    def _settings_yaml(self) -> str:
+        """Render the settings.yaml body for the family under test.
+
+        Pure and host-side so the exact bytes written into the container can be
+        inspected without launching one.
+
+        The provider module is chosen by family; the base_url key is present in
+        BOTH branches and must stay that way. provider-anthropic reads base_url
+        from CONFIG ONLY, and provider-openai likewise reads it only from
+        config (its AsyncOpenAI env fallback is an SDK accident, not a
+        contract). Omitting this key was silently sending this arm to the
+        vendor's public endpoint while the other arms honored the proxy, i.e.
+        benchmarking a different endpoint.
+
+        The Anthropic branch keeps `enable_1m_context` / `enable_prompt_caching`;
+        the OpenAI branch drops them. provider-openai does not consume either --
+        they would draw an unknown-key warning and then sit inert, implying a
+        caching posture the run does not actually have.
+
+        The OpenAI branch adds `reasoning_effort` (see `providers.REASONING_EFFORT`).
+        It is OpenAI-only: the Anthropic side reasons on a token budget, not an
+        effort level, and pinning that is a separate change.
+        """
+        family = self.provider_family
+        settings = (
+            "config:\n"
+            "  providers:\n"
+            f"    - module: {FOUNDATION_PROVIDER_MODULE[family]}\n"
+            f"      source: {FOUNDATION_PROVIDER_SOURCE[family]}\n"
+            "      config:\n"
+            f"        api_key: ${{{API_KEY_ENV}}}\n"
+            f"        base_url: ${{{BASE_URL_ENV}}}\n"
+            f"        default_model: {self.model}\n"
+        )
+        if family == ANTHROPIC:
+            settings += "        enable_1m_context: 'true'\n"
+            settings += "        enable_prompt_caching: 'true'\n"
+        elif family == OPENAI:
+            # Unquoted on purpose: YAML reads a bare `high` as the string
+            # "high", which is exactly what provider-openai validates against
+            # at mount time. Pinned so this arm runs at the same effort as the
+            # other two rather than at whatever the model defaults to.
+            settings += f"        reasoning_effort: {REASONING_EFFORT}\n"
+        settings += "        priority: 1\n"
+        # NOTE: no routing.matrix key -- it re-introduces role-based model fan-out,
+        # which would destroy the single-model-under-test premise of the benchmark.
+        # NOTE: no `bundle:` key -- the run command pins the bundle explicitly.
+        return settings
+
     async def _write_settings(self, environment: BaseEnvironment) -> None:
         """Write settings.yaml at RUNTIME, never at install time.
 
         Install steps become Docker layers: baking the API key there would put
         the secret in the image and make the install fingerprint key-dependent
         (defeating layer caching across runs).
         """
+        family = self.provider_family
         env = self.agent_env()
-        api_key = self._get_env("ANTHROPIC_API_KEY") or ""
-        env[API_KEY_ENV] = api_key
-        # provider-anthropic reads base_url from CONFIG ONLY -- it has no direct
-        # ANTHROPIC_BASE_URL fallback at runtime. Omitting this key was silently
-        # sending this arm to api.anthropic.com while the other arms honored the
-        # proxy, i.e. benchmarking a different endpoint.
-        base_url = self._get_env("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
+        env[API_KEY_ENV] = self._get_env(API_KEY_VAR[family]) or ""
+        base_url = self.base_url()
         env[BASE_URL_ENV] = base_url
 
         # Unquoted heredoc so ${API_KEY_ENV} expands in the container -- the key
         # never appears in argv or in the logged command.
-        settings = (
-            "config:\n"
-            "  providers:\n"
-            "    - module: provider-anthropic\n"
-            "      source: git+https://github.com/microsoft/"
-            "amplifier-module-provider-anthropic@main\n"
-            "      config:\n"
-            f"        api_key: ${{{API_KEY_ENV}}}\n"
-            f"        base_url: ${{{BASE_URL_ENV}}}\n"
-            f"        default_model: {self.model}\n"
-            "        enable_1m_context: 'true'\n"
-            "        enable_prompt_caching: 'true'\n"
-            "        priority: 1\n"
-        )
-        # NOTE: no routing.matrix key -- it re-introduces role-based fan-out to opus.
-        # NOTE: no `bundle:` key -- the run command pins the bundle explicitly.
+        settings = self._settings_yaml()
         command = (
             'mkdir -p "$HOME/.amplifier" && '
             f'cat > "{SETTINGS_PATH}" <<PIER_SETTINGS_EOF\n{settings}PIER_SETTINGS_EOF'
         )
         await self.exec_as_root(environment, self._wrap(command), env=env)
         self.logger.info(
-            f"Wrote amplifier settings.yaml at runtime (provider-anthropic, base_url={base_url})."
+            f"Wrote amplifier settings.yaml at runtime "
+            f"({FOUNDATION_PROVIDER_MODULE[family]}, base_url={base_url})."
         )
 
     def run_command(self, instruction_path: str) -> str:

.amplifier/evaluation/deep-swe/src/deepswe_agents/base.py

@@ -38,6 +38,13 @@
     normalize_metrics,
     normalize_opencode_metrics,
 )
+from deepswe_agents.providers import (
+    API_KEY_VAR,
+    BASE_URL_VAR,
+    DEFAULT_API_HOST,
+    DEFAULT_BASE_URL,
+    provider_family,
+)
 
 # Container path the task repo lives at. deep-swe grades a git diff of this dir.
 WORKDIR = "/app"
@@ -225,14 +232,37 @@ def model(self) -> str:
         """Bare model id, accepting both ``anthropic/claude-sonnet-5`` and ``claude-sonnet-5``."""
         return self._parsed_model_name or self.DEFAULT_MODEL
 
+    @property
+    def provider_family(self) -> str:
+        """``"anthropic"`` or ``"openai"``, derived from the model under test.
+
+        pier strips the ``<provider>/`` prefix before the adapter sees it, so
+        the bare model id is the only signal. See `deepswe_agents.providers`.
+        """
+        return provider_family(self.model)
+
+    def base_url(self) -> str:
+        """Endpoint for the selected family: host override, else the default."""
+        family = self.provider_family
+        return self._get_env(BASE_URL_VAR[family]) or DEFAULT_BASE_URL[family]
+
     def agent_env(self) -> dict[str, str]:
-        """Env for every exec: explicit PATH plus the Anthropic credentials."""
+        """Env for every exec: explicit PATH plus the selected family's credentials.
+
+        Only the family under test is forwarded. Handing the container the other
+        family's key would be dead weight at best and, for an agent that
+        auto-detects a provider from the environment, an active way to benchmark
+        a model nobody asked for.
+        """
+        family = self.provider_family
         base: dict[str, str | None] = {
             "PATH": CONTAINER_PATH,
             "HOME": "/root",
-            "ANTHROPIC_API_KEY": self._get_env("ANTHROPIC_API_KEY"),
-            "ANTHROPIC_BASE_URL": self._get_env("ANTHROPIC_BASE_URL"),
+            API_KEY_VAR[family]: self._get_env(API_KEY_VAR[family]),
+            BASE_URL_VAR[family]: self._get_env(BASE_URL_VAR[family]),
         }
+        # build_process_env drops None values, so an unset override simply does
+        # not reach the container and each client falls back to its own default.
         return self.build_process_env(base)
 
     def _wrap(self, command: str) -> str:
@@ -255,13 +285,22 @@ def install_spec(self):
         )
 
     def network_allowlist(self) -> NetworkAllowlist:
+        """Egress hosts for the family under test. LOAD-BEARING.
+
+        deep-swe tasks run with `network_mode = "no-network"`, so pier enforces
+        this list at an egress proxy. A missing API host does not degrade the
+        run -- every request is blocked, and the trial fails with a network
+        error that reads like a model or auth problem. pier's `--ae` flag adds
+        job-level env, not allowlist entries, and cannot substitute for this.
+        """
+        family = self.provider_family
         domains = []
-        base_url = self._get_env("ANTHROPIC_BASE_URL")
+        base_url = self._get_env(BASE_URL_VAR[family])
         if base_url:
             host = urlparse(base_url).hostname
             if host:
                 domains.append(host)
-        domains.append("api.anthropic.com")
+        domains.append(DEFAULT_API_HOST[family])
         if self._local_source:
             # The local-source install runs at RUNTIME (behind the egress proxy)
             # and still resolves dependencies from PyPI/GitHub.

.amplifier/evaluation/deep-swe/src/deepswe_agents/metrics.py

@@ -216,18 +216,33 @@ def as_dict(self) -> dict[str, Any]:
 
 #: USD per 1M tokens, keyed by model id. THE single rate card for this harness.
 #:
-#: Mirrors `_RATES` in amplifier-module-provider-anthropic/_cost.py, which is
-#: what stamps `cost_usd` into the amplifier arms' events.jsonl. Arms are only
-#: comparable if every dollar figure comes from the same card, so this is also
-#: what the opencode arm's cost is RECOMPUTED with -- see
-#: `compute_cost_from_tokens` and the WHY in `parse_opencode_db`.
+#: Mirrors the `_RATES` table of whichever provider module stamps `cost_usd`
+#: into the amplifier arms' events.jsonl. It now STRADDLES TWO upstream tables
+#: -- amplifier-module-provider-anthropic/_cost.py for the `claude-*` rows and
+#: amplifier-module-provider-openai/_cost.py for the `gpt-*` rows -- so a rate
+#: change in either upstream has to be reflected here. Arms are only comparable
+#: if every dollar figure comes from the same card, which is also why the
+#: opencode arm's cost is RECOMPUTED with it -- see `compute_cost_from_tokens`
+#: and the WHY in `parse_opencode_db`.
+#:
+#: LIMITATION -- long-context re-rating is not expressible here. The gpt-5.6
+#: family re-rates the WHOLE request at a higher table once input exceeds 272K
+#: tokens (for `gpt-5.6-terra`: input 5.00, output 22.50, cache_read 0.50,
+#: cache_write 6.25 -- 2x the short rates below for input/cache_read/
+#: cache_write, but 1.5x for output). This card is flat, so a
+#: recomputed figure for a long-context request is an UNDER-estimate. Practical
+#: effect: the opencode arm's cost is a FLOOR on a gpt-5.6 run, while the two
+#: amplifier arms read exact cost straight off the provider's own events and
+#: are unaffected.
 #:
 #: `opencode_vanilla.py` imports this to populate the model's `cost` block in
 #: opencode.json. One definition, one home.
 MODEL_RATES_PER_M: dict[str, dict[str, float]] = {
     "claude-sonnet-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75},
     "claude-sonnet-4-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75},
     "claude-opus-5": {"input": 5.00, "output": 25.00, "cache_read": 0.50, "cache_write": 6.25},
+    # SHORT-context rates (<=272K input tokens). See the LIMITATION note above.
+    "gpt-5.6-terra": {"input": 2.50, "output": 15.00, "cache_read": 0.25, "cache_write": 3.125},
 }