From 7d9d2eafc4d3d523e96ff5f8474d9c06483bcfd2 Mon Sep 17 00:00:00 2001 From: DavidKoleczek <45405824+DavidKoleczek@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:11:44 -0400 Subject: [PATCH] feat(evaluation): support OpenAI models in jobbench and deep-swe harnesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both harnesses were Anthropic-only on the model-under-test path. Provider selection is now derived from --model at runtime, so an OpenAI model and claude-sonnet-5 can both be run from the same tree. Per harness: - New providers.py maps a model id to a provider family and carries the per-family constants (credential env vars, provider module, opencode provider id and npm package). - The amplifier-agent arm selects the provider module by family and pins base_url explicitly rather than relying on SDK environment fallback. - The amplifier-foundation arm emits a family-appropriate settings.yaml, dropping the Anthropic-only caching and long-context keys on the OpenAI path, where they are not consumed. - The opencode-vanilla arm uses the family's ai-sdk package and provider id, with the base URL in provider options. The Anthropic-specific /v1 URL normalisation no longer runs for OpenAI, which already carries it. - Rate cards gain gpt-5.6-terra so recomputed cost is a real number rather than "not available". deep-swe additionally: - agent_env() forwards only the selected family's credentials. - network_allowlist() opens the selected family's API host. This is load-bearing: tasks run no-network behind an egress proxy, so without it every request is blocked before it reaches the model. - The preflight credential check keys off the selected family, so an OpenAI run no longer requires an unused Anthropic key. jobbench additionally: - The trial launch profile passes OpenAI credentials through to the container. Reasoning effort is pinned to "high" for all three arms on the OpenAI path, from a single REASONING_EFFORT constant per harness. The arms previously inherited differing defaults, which made reasoning-token volume incomparable across stacks. The claude-sonnet-5 path is unchanged: generated configuration is byte-identical to before for every arm. 🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- .amplifier/evaluation/deep-swe/run.py | 16 +- .../src/deepswe_agents/amplifier_agent.py | 23 ++- .../deepswe_agents/amplifier_foundation.py | 90 +++++++++--- .../deep-swe/src/deepswe_agents/base.py | 49 +++++- .../deep-swe/src/deepswe_agents/metrics.py | 25 +++- .../src/deepswe_agents/opencode_vanilla.py | 73 +++++++-- .../deep-swe/src/deepswe_agents/providers.py | 111 ++++++++++++++ .amplifier/evaluation/jobbench/README.md | 28 +++- .../jobbench/profiles/task.template.yaml | 9 ++ .../src/jobbench/agents/amplifier_agent.py | 32 +++- .../jobbench/agents/amplifier_foundation.py | 102 +++++++++---- .../src/jobbench/agents/opencode_vanilla.py | 119 ++++++++++++--- .../jobbench/src/jobbench/metrics.py | 24 ++- .../jobbench/src/jobbench/providers.py | 139 ++++++++++++++++++ 14 files changed, 719 insertions(+), 121 deletions(-) create mode 100644 .amplifier/evaluation/deep-swe/src/deepswe_agents/providers.py create mode 100644 .amplifier/evaluation/jobbench/src/jobbench/providers.py diff --git a/.amplifier/evaluation/deep-swe/run.py b/.amplifier/evaluation/deep-swe/run.py index 303485eb..962806ad 100644 --- a/.amplifier/evaluation/deep-swe/run.py +++ b/.amplifier/evaluation/deep-swe/run.py @@ -23,6 +23,7 @@ sys.path.insert(0, str(Path(__file__).parent / "src")) from deepswe_agents import AGENTS, LOCAL_SOURCE_AGENTS +from deepswe_agents.providers import API_KEY_VAR, provider_family DEEP_SWE_REPO = "https://github.com/datacurve-ai/deep-swe" DEEP_SWE_SHA = "435ee89ec2f2e2289f33b0da4f992f0b7b7266b9" @@ -128,14 +129,21 @@ def list_task_names(tasks: Path) -> list[str]: # ---------------------------------------------------------------------- -def preflight(require_docker: bool = True) -> None: +def preflight(model: str, require_docker: bool = True) -> None: pier = shutil.which("pier") if not pier: die(f"`pier` is not on PATH. Install it with:\n {PIER_INSTALL_CMD}") check_pier_is_git_build(pier) - if not os.environ.get("ANTHROPIC_API_KEY"): - die("ANTHROPIC_API_KEY is not set. Export it before running.") + # Only the family actually under test is required. Demanding an Anthropic + # key for an OpenAI run would force a dummy value whose only effect is to + # satisfy this check -- a gate that has stopped gating anything. + # `--model` may carry a `/` prefix; the family is derived from the + # bare id, exactly as every adapter derives it. + bare_model = model.split("/", 1)[-1] + key_var = API_KEY_VAR[provider_family(bare_model)] + if not os.environ.get(key_var): + die(f"{key_var} is not set (required for model {model!r}). Export it before running.") if require_docker: if not shutil.which("docker"): @@ -627,7 +635,7 @@ def main(argv: list[str]) -> int: die(f"--local-source path does not exist: {src}") args.local_source = str(src) - preflight(require_docker=not args.dry_run) + preflight(args.model, require_docker=not args.dry_run) tasks = ensure_tasks(tasks_dir) # Resolve ONCE, then hand the identical explicit list to every agent. diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_agent.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_agent.py index 5de75de7..f33bd25e 100644 --- a/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_agent.py +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_agent.py @@ -15,6 +15,7 @@ AmplifierBaseAgent, _as_bool, ) +from deepswe_agents.providers import AGENT_PROVIDER_MODULE, OPENAI, REASONING_EFFORT HOST_CONFIG_PATH = "/root/host-config.json" @@ -64,11 +65,29 @@ async def setup(self, environment: BaseEnvironment) -> None: ) def _host_config(self) -> str: + # `provider.module` is what SELECTS the provider, not merely what + # configures it: single_turn reads it and hands the name to + # inject_provider, which mounts that module alone. The friendly names + # ("anthropic", "openai") are the ones its config merger accepts. + family = self.provider_family + provider_config: dict[str, Any] = {"default_model": self.model} + if family == OPENAI: + # provider-openai reads base_url from CONFIG ONLY. It does have an + # OPENAI_BASE_URL fallback, but only as an incidental behavior of + # AsyncOpenAI's own constructor -- not a contract this module reads. + # Stating it here is what makes the endpoint under benchmark + # explicit rather than an artifact of the SDK. + provider_config["base_url"] = self.base_url() + # Pin the reasoning effort rather than inheriting the model's own + # default, so this arm is benchmarked at the same effort as the + # other two. provider-openai validates the value at mount, so a + # typo fails the run loudly instead of silently reverting. + provider_config["reasoning_effort"] = REASONING_EFFORT config: dict[str, Any] = { "approval": {"mode": "yes"}, "provider": { - "module": "anthropic", - "config": {"default_model": self.model}, + "module": AGENT_PROVIDER_MODULE[family], + "config": provider_config, }, } if self._raw_llm_payloads: diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_foundation.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_foundation.py index 1330bdf3..e497043c 100644 --- a/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_foundation.py +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/amplifier_foundation.py @@ -11,11 +11,22 @@ from pier.models.agent.network import NetworkAllowlist from deepswe_agents.base import UV_PRELUDE, WORKDIR, AmplifierBaseAgent +from deepswe_agents.providers import ( + ANTHROPIC, + API_KEY_VAR, + FOUNDATION_PROVIDER_MODULE, + FOUNDATION_PROVIDER_SOURCE, + OPENAI, + REASONING_EFFORT, +) SETTINGS_PATH = "$HOME/.amplifier/settings.yaml" # Env vars used to smuggle provider config into the heredoc without it appearing # in argv. The key must never be logged; the base URL rides along for symmetry. +# Deliberately provider-neutral names: the SOURCE var differs by family +# (ANTHROPIC_API_KEY vs OPENAI_API_KEY) but the smuggling channel does not, so +# the settings template has one shape regardless of which model is under test. API_KEY_ENV = "AMPLIFIER_BENCH_API_KEY" BASE_URL_ENV = "AMPLIFIER_BENCH_BASE_URL" @@ -118,6 +129,55 @@ async def setup(self, environment: BaseEnvironment) -> None: await super().setup(environment) await self._write_settings(environment) + def _settings_yaml(self) -> str: + """Render the settings.yaml body for the family under test. + + Pure and host-side so the exact bytes written into the container can be + inspected without launching one. + + The provider module is chosen by family; the base_url key is present in + BOTH branches and must stay that way. provider-anthropic reads base_url + from CONFIG ONLY, and provider-openai likewise reads it only from + config (its AsyncOpenAI env fallback is an SDK accident, not a + contract). Omitting this key was silently sending this arm to the + vendor's public endpoint while the other arms honored the proxy, i.e. + benchmarking a different endpoint. + + The Anthropic branch keeps `enable_1m_context` / `enable_prompt_caching`; + the OpenAI branch drops them. provider-openai does not consume either -- + they would draw an unknown-key warning and then sit inert, implying a + caching posture the run does not actually have. + + The OpenAI branch adds `reasoning_effort` (see `providers.REASONING_EFFORT`). + It is OpenAI-only: the Anthropic side reasons on a token budget, not an + effort level, and pinning that is a separate change. + """ + family = self.provider_family + settings = ( + "config:\n" + " providers:\n" + f" - module: {FOUNDATION_PROVIDER_MODULE[family]}\n" + f" source: {FOUNDATION_PROVIDER_SOURCE[family]}\n" + " config:\n" + f" api_key: ${{{API_KEY_ENV}}}\n" + f" base_url: ${{{BASE_URL_ENV}}}\n" + f" default_model: {self.model}\n" + ) + if family == ANTHROPIC: + settings += " enable_1m_context: 'true'\n" + settings += " enable_prompt_caching: 'true'\n" + elif family == OPENAI: + # Unquoted on purpose: YAML reads a bare `high` as the string + # "high", which is exactly what provider-openai validates against + # at mount time. Pinned so this arm runs at the same effort as the + # other two rather than at whatever the model defaults to. + settings += f" reasoning_effort: {REASONING_EFFORT}\n" + settings += " priority: 1\n" + # NOTE: no routing.matrix key -- it re-introduces role-based model fan-out, + # which would destroy the single-model-under-test premise of the benchmark. + # NOTE: no `bundle:` key -- the run command pins the bundle explicitly. + return settings + async def _write_settings(self, environment: BaseEnvironment) -> None: """Write settings.yaml at RUNTIME, never at install time. @@ -125,41 +185,23 @@ async def _write_settings(self, environment: BaseEnvironment) -> None: the secret in the image and make the install fingerprint key-dependent (defeating layer caching across runs). """ + family = self.provider_family env = self.agent_env() - api_key = self._get_env("ANTHROPIC_API_KEY") or "" - env[API_KEY_ENV] = api_key - # provider-anthropic reads base_url from CONFIG ONLY -- it has no direct - # ANTHROPIC_BASE_URL fallback at runtime. Omitting this key was silently - # sending this arm to api.anthropic.com while the other arms honored the - # proxy, i.e. benchmarking a different endpoint. - base_url = self._get_env("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + env[API_KEY_ENV] = self._get_env(API_KEY_VAR[family]) or "" + base_url = self.base_url() env[BASE_URL_ENV] = base_url # Unquoted heredoc so ${API_KEY_ENV} expands in the container -- the key # never appears in argv or in the logged command. - settings = ( - "config:\n" - " providers:\n" - " - module: provider-anthropic\n" - " source: git+https://github.com/microsoft/" - "amplifier-module-provider-anthropic@main\n" - " config:\n" - f" api_key: ${{{API_KEY_ENV}}}\n" - f" base_url: ${{{BASE_URL_ENV}}}\n" - f" default_model: {self.model}\n" - " enable_1m_context: 'true'\n" - " enable_prompt_caching: 'true'\n" - " priority: 1\n" - ) - # NOTE: no routing.matrix key -- it re-introduces role-based fan-out to opus. - # NOTE: no `bundle:` key -- the run command pins the bundle explicitly. + settings = self._settings_yaml() command = ( 'mkdir -p "$HOME/.amplifier" && ' f'cat > "{SETTINGS_PATH}" < str: diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/base.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/base.py index e4088082..94e4bcc0 100644 --- a/.amplifier/evaluation/deep-swe/src/deepswe_agents/base.py +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/base.py @@ -38,6 +38,13 @@ normalize_metrics, normalize_opencode_metrics, ) +from deepswe_agents.providers import ( + API_KEY_VAR, + BASE_URL_VAR, + DEFAULT_API_HOST, + DEFAULT_BASE_URL, + provider_family, +) # Container path the task repo lives at. deep-swe grades a git diff of this dir. WORKDIR = "/app" @@ -225,14 +232,37 @@ def model(self) -> str: """Bare model id, accepting both ``anthropic/claude-sonnet-5`` and ``claude-sonnet-5``.""" return self._parsed_model_name or self.DEFAULT_MODEL + @property + def provider_family(self) -> str: + """``"anthropic"`` or ``"openai"``, derived from the model under test. + + pier strips the ``/`` prefix before the adapter sees it, so + the bare model id is the only signal. See `deepswe_agents.providers`. + """ + return provider_family(self.model) + + def base_url(self) -> str: + """Endpoint for the selected family: host override, else the default.""" + family = self.provider_family + return self._get_env(BASE_URL_VAR[family]) or DEFAULT_BASE_URL[family] + def agent_env(self) -> dict[str, str]: - """Env for every exec: explicit PATH plus the Anthropic credentials.""" + """Env for every exec: explicit PATH plus the selected family's credentials. + + Only the family under test is forwarded. Handing the container the other + family's key would be dead weight at best and, for an agent that + auto-detects a provider from the environment, an active way to benchmark + a model nobody asked for. + """ + family = self.provider_family base: dict[str, str | None] = { "PATH": CONTAINER_PATH, "HOME": "/root", - "ANTHROPIC_API_KEY": self._get_env("ANTHROPIC_API_KEY"), - "ANTHROPIC_BASE_URL": self._get_env("ANTHROPIC_BASE_URL"), + API_KEY_VAR[family]: self._get_env(API_KEY_VAR[family]), + BASE_URL_VAR[family]: self._get_env(BASE_URL_VAR[family]), } + # build_process_env drops None values, so an unset override simply does + # not reach the container and each client falls back to its own default. return self.build_process_env(base) def _wrap(self, command: str) -> str: @@ -255,13 +285,22 @@ def install_spec(self): ) def network_allowlist(self) -> NetworkAllowlist: + """Egress hosts for the family under test. LOAD-BEARING. + + deep-swe tasks run with `network_mode = "no-network"`, so pier enforces + this list at an egress proxy. A missing API host does not degrade the + run -- every request is blocked, and the trial fails with a network + error that reads like a model or auth problem. pier's `--ae` flag adds + job-level env, not allowlist entries, and cannot substitute for this. + """ + family = self.provider_family domains = [] - base_url = self._get_env("ANTHROPIC_BASE_URL") + base_url = self._get_env(BASE_URL_VAR[family]) if base_url: host = urlparse(base_url).hostname if host: domains.append(host) - domains.append("api.anthropic.com") + domains.append(DEFAULT_API_HOST[family]) if self._local_source: # The local-source install runs at RUNTIME (behind the egress proxy) # and still resolves dependencies from PyPI/GitHub. diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/metrics.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/metrics.py index 5656ad1d..1d4cdc3a 100644 --- a/.amplifier/evaluation/deep-swe/src/deepswe_agents/metrics.py +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/metrics.py @@ -216,11 +216,24 @@ def as_dict(self) -> dict[str, Any]: #: USD per 1M tokens, keyed by model id. THE single rate card for this harness. #: -#: Mirrors `_RATES` in amplifier-module-provider-anthropic/_cost.py, which is -#: what stamps `cost_usd` into the amplifier arms' events.jsonl. Arms are only -#: comparable if every dollar figure comes from the same card, so this is also -#: what the opencode arm's cost is RECOMPUTED with -- see -#: `compute_cost_from_tokens` and the WHY in `parse_opencode_db`. +#: Mirrors the `_RATES` table of whichever provider module stamps `cost_usd` +#: into the amplifier arms' events.jsonl. It now STRADDLES TWO upstream tables +#: -- amplifier-module-provider-anthropic/_cost.py for the `claude-*` rows and +#: amplifier-module-provider-openai/_cost.py for the `gpt-*` rows -- so a rate +#: change in either upstream has to be reflected here. Arms are only comparable +#: if every dollar figure comes from the same card, which is also why the +#: opencode arm's cost is RECOMPUTED with it -- see `compute_cost_from_tokens` +#: and the WHY in `parse_opencode_db`. +#: +#: LIMITATION -- long-context re-rating is not expressible here. The gpt-5.6 +#: family re-rates the WHOLE request at a higher table once input exceeds 272K +#: tokens (for `gpt-5.6-terra`: input 5.00, output 22.50, cache_read 0.50, +#: cache_write 6.25 -- 2x the short rates below for input/cache_read/ +#: cache_write, but 1.5x for output). This card is flat, so a +#: recomputed figure for a long-context request is an UNDER-estimate. Practical +#: effect: the opencode arm's cost is a FLOOR on a gpt-5.6 run, while the two +#: amplifier arms read exact cost straight off the provider's own events and +#: are unaffected. #: #: `opencode_vanilla.py` imports this to populate the model's `cost` block in #: opencode.json. One definition, one home. @@ -228,6 +241,8 @@ def as_dict(self) -> dict[str, Any]: "claude-sonnet-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75}, "claude-sonnet-4-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75}, "claude-opus-5": {"input": 5.00, "output": 25.00, "cache_read": 0.50, "cache_write": 6.25}, + # SHORT-context rates (<=272K input tokens). See the LIMITATION note above. + "gpt-5.6-terra": {"input": 2.50, "output": 15.00, "cache_read": 0.25, "cache_write": 3.125}, } diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/opencode_vanilla.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/opencode_vanilla.py index 261bc3a5..774384c6 100644 --- a/.amplifier/evaluation/deep-swe/src/deepswe_agents/opencode_vanilla.py +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/opencode_vanilla.py @@ -1,4 +1,8 @@ -"""Stock OpenCode talking straight to Anthropic. The control arm.""" +"""Stock OpenCode talking straight to the model vendor. The control arm. + +Provider (Anthropic or OpenAI) is derived from the model under test; see +`deepswe_agents.providers`. +""" from __future__ import annotations @@ -10,6 +14,13 @@ from deepswe_agents.base import OPENCODE_PRELUDE, AmplifierBaseAgent from deepswe_agents.metrics import MODEL_RATES_PER_M +from deepswe_agents.providers import ( + ANTHROPIC, + OPENAI, + OPENCODE_NPM, + OPENCODE_PROVIDER_ID, + REASONING_EFFORT, +) # The `cost` block written into opencode.json is BEST EFFORT only: opencode # ignores the `cost.cache` override, so the published dollar figure comes from @@ -40,7 +51,8 @@ def name() -> str: def agent_env(self) -> dict[str, str]: """Normalize ANTHROPIC_BASE_URL for opencode's ai-sdk provider. - The two clients disagree on what "base URL" means: + ANTHROPIC-ONLY. The two Anthropic clients disagree on what "base URL" + means: * the Anthropic SDK (amplifier-agent) wants the host root and appends `/v1` itself -> https://api.anthropic.com * ai-sdk `@ai-sdk/anthropic` (opencode) treats it as the full API root @@ -50,8 +62,14 @@ def agent_env(self) -> dict[str, str]: `https://api.anthropic.com/messages`, which 404s. opencode reports that as a bare `Error: Not Found` and aborts the run -- with no mention of a URL, which makes it look like a model or auth problem. + + OPENAI_BASE_URL has no such mismatch: both sides already mean the `/v1` + API root, so the OpenAI family is passed through untouched. Applying + this fixup there would append a second `/v1` and break every request. """ env = super().agent_env() + if self.provider_family != ANTHROPIC: + return env base = env.get("ANTHROPIC_BASE_URL") if base: trimmed = base.rstrip("/") @@ -68,28 +86,48 @@ def _model_entry(self) -> dict[str, Any]: "output": rates["output"], "cache": {"read": rates["cache_read"], "write": rates["cache_write"]}, } + if self.provider_family == OPENAI: + # MODEL-level, not provider-level. The provider `options` block + # (which carries baseURL) does NOT reach the wire with this key -- + # only `provider..models..options.reasoningEffort` does. + # + # This overrides opencode's own built-in default, which stamps + # `reasoningEffort: "medium"` onto any model id containing "gpt-5". + # Without this the arm would silently benchmark medium while the + # other two ran at REASONING_EFFORT. Only that one default is + # replaced; opencode's other gpt-5 defaults still apply. + entry["reasoning"] = True + entry["options"] = {"reasoningEffort": REASONING_EFFORT} return entry def _opencode_config(self) -> str: model = self.model + family = self.provider_family + provider_id = OPENCODE_PROVIDER_ID[family] + provider: dict[str, Any] = { + "npm": OPENCODE_NPM[family], + "models": {model: self._model_entry()}, + } + if family == OPENAI: + # opencode reads the endpoint from provider..options.baseURL, + # NOT from the provider root. Same shape pier's own opencode + # adapter writes (pier/agents/installed/opencode.py). Put at the + # root it is silently ignored and every request goes to the public + # OpenAI endpoint instead of the one under benchmark. + provider["options"] = {"baseURL": self.base_url()} return json.dumps( { "$schema": "https://opencode.ai/config.json", - "model": f"anthropic/{model}", + "model": f"{provider_id}/{model}", # Pin the SMALL model to the benchmark model. opencode uses a # separate "small" model for the session-title agent, and its - # default family priority ends at claude-haiku -- a model this - # endpoint does not serve. That request fails with a bare - # `AI_APICallError: Not Found` and kills the process (exit 1) - # before any task work happens. It was the single most common - # failure of this arm. - "small_model": f"anthropic/{model}", - "provider": { - "anthropic": { - "npm": "@ai-sdk/anthropic", - "models": {model: self._model_entry()}, - } - }, + # default family priority ends at a cheap model this endpoint + # may not serve (claude-haiku on the Anthropic side). That + # request fails with a bare `AI_APICallError: Not Found` and + # kills the process (exit 1) before any task work happens. It + # was the single most common failure of this arm. + "small_model": f"{provider_id}/{model}", + "provider": {provider_id: provider}, } ) @@ -149,4 +187,7 @@ def run_command(self, instruction_path: str) -> str: # banner, and the agent never runs. yargs is configured with # `populate--: true`, and opencode's run command merges `argv["--"]` back # into the message, so the prompt still arrives intact. - return f'opencode run --model anthropic/{self.model} --auto -- "$(cat {instruction_path})"' + provider_id = OPENCODE_PROVIDER_ID[self.provider_family] + return ( + f'opencode run --model {provider_id}/{self.model} --auto -- "$(cat {instruction_path})"' + ) diff --git a/.amplifier/evaluation/deep-swe/src/deepswe_agents/providers.py b/.amplifier/evaluation/deep-swe/src/deepswe_agents/providers.py new file mode 100644 index 00000000..2e0b0058 --- /dev/null +++ b/.amplifier/evaluation/deep-swe/src/deepswe_agents/providers.py @@ -0,0 +1,111 @@ +"""Which provider family a model id belongs to, and the per-family constants. + +The harness benchmarks ONE model at a time across every arm, but that model may +be served by either Anthropic or OpenAI. Every arm needs the same four facts -- +which credential env vars to forward, which API host to open in the egress +allowlist, which provider module to configure, and which opencode provider id to +write -- so they live here once instead of being re-derived (differently) in +four adapters. + +Selection is derived from the bare model id at RUNTIME. pier splits `--model` on +the first `/` and hands the adapter only the right-hand side, so the +`anthropic/` or `openai/` prefix a user types never reaches this module. The +model id itself is the only signal available, which is why the test is on the +`gpt-` prefix rather than on a provider label. +""" + +from __future__ import annotations + +OPENAI = "openai" +ANTHROPIC = "anthropic" + + +def provider_family(model: str) -> str: + """Return ``"openai"`` or ``"anthropic"`` for a bare model id. + + Anthropic is the default because it is the harness's historical single + provider: an unrecognised id keeps every arm on exactly the path it took + before OpenAI support existed, rather than failing closed on a new one. + + >>> provider_family("claude-sonnet-5") + 'anthropic' + >>> provider_family("gpt-5.6-terra") + 'openai' + """ + return OPENAI if (model or "").startswith("gpt-") else ANTHROPIC + + +#: Credential env var forwarded into the container, per family. +API_KEY_VAR = { + ANTHROPIC: "ANTHROPIC_API_KEY", + OPENAI: "OPENAI_API_KEY", +} + +#: Endpoint override env var read on the HOST, per family. +BASE_URL_VAR = { + ANTHROPIC: "ANTHROPIC_BASE_URL", + OPENAI: "OPENAI_BASE_URL", +} + +#: Fallback endpoint when the host sets no override. Note the shapes differ: +#: the Anthropic SDK wants the host root and appends `/v1` itself, while the +#: OpenAI SDK wants the API root INCLUDING `/v1`. +DEFAULT_BASE_URL = { + ANTHROPIC: "https://api.anthropic.com", + OPENAI: "https://api.openai.com/v1", +} + +#: Host always opened in the egress allowlist for the family, on top of the +#: parsed hostname of any BASE_URL override. deep-swe tasks run with +#: `network_mode = "no-network"`, so an absent host here is not a slow request +#: -- it is a proxy-blocked one, reported as a network error rather than a +#: model error. +DEFAULT_API_HOST = { + ANTHROPIC: "api.anthropic.com", + OPENAI: "api.openai.com", +} + +#: Reasoning effort pinned for the OPENAI family, benchmark-wide. This single +#: value is the pin for all three arms -- amplifier-agent, foundation, and +#: opencode-vanilla each spell it differently on the wire +#: (`reasoning_effort` in host-config.json, `reasoning_effort` in settings.yaml, +#: `options.reasoningEffort` in opencode.json), but all three read it from here, +#: so re-pinning the benchmark is a one-line change. +#: +#: Stated EXPLICITLY rather than left to defaults: opencode silently applies +#: `reasoningEffort: "medium"` to any model id containing "gpt-5", so an +#: unpinned run would benchmark the arms against different effort levels. +#: +#: OPENAI ONLY. The Anthropic family uses a thinking-token BUDGET instead, which +#: this constant does not express and which is deliberately not pinned here -- +#: adding it to an Anthropic branch would be a different mechanism wearing the +#: same name. +REASONING_EFFORT = "high" + +#: Provider module name used in `amplifier-agent`'s host-config.json. These are +#: the friendly names its config merger accepts (`_PROVIDER_NAME_TO_MODULE_KEY`). +AGENT_PROVIDER_MODULE = { + ANTHROPIC: "anthropic", + OPENAI: "openai", +} + +#: Provider module + git source written into foundation's settings.yaml. +FOUNDATION_PROVIDER_MODULE = { + ANTHROPIC: "provider-anthropic", + OPENAI: "provider-openai", +} +FOUNDATION_PROVIDER_SOURCE = { + ANTHROPIC: "git+https://github.com/microsoft/amplifier-module-provider-anthropic@main", + OPENAI: "git+https://github.com/microsoft/amplifier-module-provider-openai@main", +} + +#: opencode provider id (also the `/` prefix) and the ai-sdk +#: npm package that backs it. +OPENCODE_PROVIDER_ID = { + ANTHROPIC: "anthropic", + OPENAI: "openai", +} +OPENCODE_NPM = { + ANTHROPIC: "@ai-sdk/anthropic", + OPENAI: "@ai-sdk/openai", +} diff --git a/.amplifier/evaluation/jobbench/README.md b/.amplifier/evaluation/jobbench/README.md index 5ccc4bf1..858a7de7 100644 --- a/.amplifier/evaluation/jobbench/README.md +++ b/.amplifier/evaluation/jobbench/README.md @@ -62,15 +62,20 @@ extractors (pandas, openpyxl, xlrd, python-pptx, pdfplumber, mammoth). Environment variables: ``` -ANTHROPIC_API_KEY required. Passed into each trial container by - profiles/task.template.yaml passthrough.services; the - value never enters this Python process. -ANTHROPIC_BASE_URL required by the same passthrough block. +ANTHROPIC_API_KEY required when --model is a claude-* model. Passed into + each trial container by profiles/task.template.yaml + passthrough.services; the value never enters this Python + process. +ANTHROPIC_BASE_URL required by the same passthrough block, same condition. OPENAI_API_KEY required to grade. src/jobbench/grading.py reads it and passes it to the judge as --api-key. Override per - invocation with --judge-api-key. + invocation with --judge-api-key. ALSO the agent-under- + test's key when --model is a gpt-* model, via the same + passthrough block as above. OPENAI_BASE_URL the judge endpoint, read the same way. Override with - --judge-api-base. + --judge-api-base. ALSO the agent-under-test's endpoint + for gpt-* models, where it is required (no default) -- + unlike ANTHROPIC_BASE_URL it must already end in /v1. JOBBENCH_CACHE_DIR optional. Moves the dataset cache off the default dataset-cache/ so one download is shared across checkouts. @@ -133,6 +138,11 @@ python run.py run --agent all --all-tasks --split main \ Useful flags: `--model`, `--bundle` (amplifier-foundation only), `--timeout`, `--output-dir`, `--no-grade`, and the `--judge-*` family. +`--model` also picks the provider: `gpt-*` routes every arm to OpenAI, +anything else to Anthropic (`src/jobbench/providers.py`). Nothing else needs +changing to swap families -- the launch profile passes both key/base-url pairs +through and the DTU engine forwards only the ones actually set on the host. + ### Cost and runtime Pilot on a single task before committing to a sweep. `--dry-run` prints the @@ -335,3 +345,9 @@ on cache rates. If a session's model is not in this harness's rate card, `cost_usd` for that session is `"not_available"` even though opencode itself reported a number -- that number is left out because it is not comparable to the amplifier arms' figures, not because it does not exist. + +The card is flat, so it cannot express a context-length threshold. For +`gpt-5.6-terra`, whose upstream rates re-rate above 272K input tokens, this +arm's `cost_usd` is therefore a FLOOR: any session that crosses the threshold +is understated. The amplifier arms are unaffected -- their `cost_usd` comes +from provider events, priced upstream with the real tiering. diff --git a/.amplifier/evaluation/jobbench/profiles/task.template.yaml b/.amplifier/evaluation/jobbench/profiles/task.template.yaml index b34e269b..0cb67605 100644 --- a/.amplifier/evaluation/jobbench/profiles/task.template.yaml +++ b/.amplifier/evaluation/jobbench/profiles/task.template.yaml @@ -18,11 +18,20 @@ base: passthrough: allow_external: true # tasks may require live web search (main split) + # Both provider families are listed unconditionally: the DTU engine only + # exports an entry whose key_env is actually set on the host, so listing the + # OpenAI pair costs nothing on an Anthropic-only host and vice versa. Which + # one an arm uses is decided at trial time from --model (see + # src/jobbench/providers.py), not here. services: - name: anthropic key_env: ANTHROPIC_API_KEY - name: anthropic_base_url key_env: ANTHROPIC_BASE_URL + - name: openai + key_env: OPENAI_API_KEY + - name: openai_base_url + key_env: OPENAI_BASE_URL provision: setup_cmds: diff --git a/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_agent.py b/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_agent.py index 612d52b7..7de1813e 100644 --- a/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_agent.py +++ b/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_agent.py @@ -13,12 +13,14 @@ from __future__ import annotations import json +import os import shlex import uuid from jobbench import images from jobbench.agents.base import Adapter, AdapterError, register from jobbench.dtu import DTU +from jobbench.providers import OPENAI, REASONING_EFFORT, provider_family DEFAULT_MODEL = "claude-sonnet-5" HOST_CONFIG_PATH = "/root/host-config.json" @@ -47,13 +49,39 @@ def __init__(self) -> None: async def configure(self, dtu: DTU, *, model: str) -> None: """Write the per-trial host-config and guarantee PATH. - No API key is written here -- ANTHROPIC_API_KEY reaches the + The provider module is derived from the model (see + jobbench.providers), so the same adapter serves both families. + + No API key is written here -- the family's key env var reaches the container through the launch profile's `passthrough.services`, so it never touches disk in plaintext under our control. """ + family = provider_family(model) + provider_config: dict[str, str] = {"default_model": model} + if family.name == OPENAI: + # provider-openai would otherwise fall back to the OpenAI SDK's + # own env/default resolution, which is invisible in the captured + # config and can silently point this arm at a different endpoint + # than the other arms. Pin it in the config instead. + # + # The base URL is not a secret (unlike the API key, which still + # only ever arrives via passthrough), so resolving it host-side + # is safe. Required, not defaulted: the host must already have it + # set for passthrough to forward it at all. + base_url = os.environ.get(family.base_url_env) + if not base_url: + raise AdapterError( + f"amplifier-agent configure failed: {family.base_url_env} is not set on " + f"the host, so model {model!r} has no endpoint to target" + ) + provider_config["base_url"] = base_url + # Benchmark-wide pin (jobbench.providers.REASONING_EFFORT). + # provider-openai validates this at mount; left unset, the model's + # own default effort applies and would differ from the other arms. + provider_config["reasoning_effort"] = REASONING_EFFORT config = { "approval": {"mode": "yes"}, - "provider": {"module": "anthropic", "config": {"default_model": model}}, + "provider": {"module": family.agent_module, "config": provider_config}, } payload = json.dumps(config) script = ( diff --git a/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_foundation.py b/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_foundation.py index b76f15fb..9cae7ed5 100644 --- a/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_foundation.py +++ b/.amplifier/evaluation/jobbench/src/jobbench/agents/amplifier_foundation.py @@ -13,7 +13,7 @@ against the CONTAINER's own environment (already populated by the launch profile's passthrough.services), not a dict we control. Either way the secret value itself never appears in our argv or logs -- only the literal - `${ANTHROPIC_API_KEY}` reference does. + reference (e.g. `${ANTHROPIC_API_KEY}`, `${OPENAI_API_KEY}`) does. The bake profile (profiles/agents/amplifier-foundation.bake.yaml) installs the CLI and pre-warms the default bundle's module resolution, with no provider, @@ -22,9 +22,12 @@ from __future__ import annotations +import os + from jobbench import images from jobbench.agents.base import Adapter, AdapterError, register from jobbench.dtu import DTU +from jobbench.providers import ANTHROPIC, OPENAI, REASONING_EFFORT, provider_family SETTINGS_PATH = "$HOME/.amplifier/settings.yaml" @@ -37,35 +40,67 @@ "#subdirectory=bundles/anchors/bundle.md" ) -# Unquoted heredoc: ${...} expands INSIDE the container against its own -# environment, so the secret value never crosses into our Python process, -# argv, or logs -- only the literal reference does. -# -# provider-anthropic reads base_url from CONFIG ONLY; it has no runtime env -# fallback. Omitting this key would silently send this arm to -# api.anthropic.com while every other arm hits the configured proxy, i.e. -# benchmarking a different endpoint. The `:-https://api.anthropic.com` default -# only fires if a launch profile forgot to pass ANTHROPIC_BASE_URL through. -# -# No routing.matrix key here (re-introduces role-based fan-out to a different -# model) and no bundle: key (the run command pins the bundle explicitly, so a -# stray bundle: entry here would never be read anyway). -_SETTINGS_TEMPLATE = """config: - providers: - - module: provider-anthropic - source: git+https://github.com/microsoft/amplifier-module-provider-anthropic@main - config: - api_key: ${ANTHROPIC_API_KEY} - base_url: ${ANTHROPIC_BASE_URL:-https://api.anthropic.com} - default_model: __MODEL__ - enable_1m_context: 'true' - enable_prompt_caching: 'true' - priority: 1 -""" - _HEREDOC_MARKER = "JOBBENCH_SETTINGS_EOF" +def _settings_yaml(model: str) -> str: + """The settings.yaml body for one model, provider family and all. + + Written for an UNQUOTED heredoc: the ${...} references below expand INSIDE + the container against its own environment, so the secret value never + crosses into our Python process, argv, or logs -- only the literal + reference does. Nothing here reads an env var host-side. + + Both provider modules read base_url from CONFIG ONLY; neither has a runtime + env fallback. Omitting the key would silently send this arm to the vendor's + public endpoint while every other arm hits the configured one, i.e. + benchmarking a different backend. The Anthropic `:-https://api.anthropic.com` + default only fires if a launch profile forgot to pass ANTHROPIC_BASE_URL + through, and is kept only because api.anthropic.com genuinely is that + family's endpoint. There is deliberately NO equivalent default for OpenAI: + the endpoint under test is not necessarily the public one, so an unset + OPENAI_BASE_URL must fail loudly rather than quietly re-target the run. + + enable_1m_context / enable_prompt_caching are provider-anthropic config + keys. They are dropped for provider-openai rather than passed inertly, so + the captured settings.yaml describes only knobs that actually exist. The + mirror image is reasoning_effort, a provider-openai key written only on + that branch: it pins the benchmark-wide effort level (see + jobbench.providers.REASONING_EFFORT) so this arm matches the other two. + Anthropic has no equivalent -- it budgets thinking tokens instead -- so + that branch is left alone deliberately, not by omission. + + No routing.matrix key in either branch (re-introduces role-based fan-out to + a different model, which would invalidate a single-model comparison) and no + bundle: key (the run command pins the bundle explicitly, so a stray bundle: + entry here would never be read anyway). + """ + family = provider_family(model) + if family.default_base_url is not None: + base_url = f"${{{family.base_url_env}:-{family.default_base_url}}}" + else: + base_url = f"${{{family.base_url_env}}}" + lines = [ + "config:", + " providers:", + f" - module: {family.foundation_module}", + f" source: {family.foundation_source}", + " config:", + f" api_key: ${{{family.api_key_env}}}", + f" base_url: {base_url}", + f" default_model: {model}", + ] + if family.name == ANTHROPIC: + lines += [ + " enable_1m_context: 'true'", + " enable_prompt_caching: 'true'", + ] + elif family.name == OPENAI: + lines.append(f" reasoning_effort: {REASONING_EFFORT}") + lines.append(" priority: 1") + return "\n".join(lines) + "\n" + + @register class AmplifierFoundationAdapter(Adapter): name = "amplifier-foundation" @@ -91,7 +126,18 @@ async def configure(self, dtu: DTU, *, model: str) -> None: would put the secret in the image. The model is also a per-run choice, so it belongs here too, not in the bake profile. """ - settings = _SETTINGS_TEMPLATE.replace("__MODEL__", model) + family = provider_family(model) + if family.default_base_url is None and family.base_url_env not in os.environ: + # Presence check only -- the VALUE is never read here; the heredoc + # below expands it inside the container instead. Without a default + # to fall back on, an unset var would expand to an empty base_url + # and surface as an opaque provider error mid-run. The host is the + # right place to check: passthrough only forwards vars set here. + raise AdapterError( + f"amplifier-foundation configure failed: {family.base_url_env} is not set " + f"on the host, so model {model!r} has no endpoint to target" + ) + settings = _settings_yaml(model) script = ( 'mkdir -p "$HOME/.amplifier" && ' f'cat > "{SETTINGS_PATH}" <<{_HEREDOC_MARKER}\n' diff --git a/.amplifier/evaluation/jobbench/src/jobbench/agents/opencode_vanilla.py b/.amplifier/evaluation/jobbench/src/jobbench/agents/opencode_vanilla.py index 8bc09919..07dbecd6 100644 --- a/.amplifier/evaluation/jobbench/src/jobbench/agents/opencode_vanilla.py +++ b/.amplifier/evaluation/jobbench/src/jobbench/agents/opencode_vanilla.py @@ -1,4 +1,4 @@ -"""Stock OpenCode talking straight to Anthropic. The control arm. +"""Stock OpenCode talking straight to the provider. The control arm. Ports deep-swe's OpencodeVanillaAgent (../deep-swe/src/deepswe_agents/opencode_vanilla.py) to jobbench's Adapter @@ -10,6 +10,11 @@ one invocation from whatever value the launch profile's passthrough.services already put in the container's environment. +Which provider opencode is pointed at follows from the model under test (see +jobbench.providers). Only the Anthropic path needs the base-URL rewrite above; +the OpenAI path pins its endpoint in opencode.json instead, because +OPENAI_BASE_URL is already the full API root ai-sdk wants. + The `cost` block written into opencode.json is best-effort only: opencode ignores the `cost.cache` override, so the published dollar figure actually comes from `metrics.parse_opencode_db`, which recomputes cost from the @@ -20,18 +25,39 @@ from __future__ import annotations import json +import os from typing import Any from jobbench import images from jobbench.agents.base import Adapter, AdapterError, register from jobbench.dtu import DTU from jobbench.metrics import MODEL_RATES_PER_M +from jobbench.providers import OPENAI, REASONING_EFFORT, provider_family OPENCODE_CONFIG_PATH = "$HOME/.config/opencode/opencode.json" _HEREDOC_MARKER = "JOBBENCH_OPENCODE_EOF" def _model_entry(model: str) -> dict[str, Any]: + """The `provider..models.` block for one model. + + The OpenAI branch pins reasoning effort here and NOWHERE ELSE in this + config. opencode merges the per-model `options` dict over its own defaults + (packages/opencode/src/session/llm/request.ts:91) and forwards the result + to the wire (transform.ts:1414); the PROVIDER-level `options` block does + not carry this key through -- only connection settings like baseURL + survive there. Left unset, opencode applies a built-in + `reasoningEffort: "medium"` to any model id containing "gpt-5" + (transform.ts:1289-1291), so this override is what keeps the control arm + at the same effort as the two amplifier arms. + + `reasoning: true` is not strictly required for options.reasoningEffort to + reach the wire (transform.ts:1364 is an OR), but it is the accurate + declaration for a reasoning model, so it is stated rather than implied. + + Anthropic models get neither key: that family budgets thinking tokens + instead, and pinning it is out of scope. + """ entry: dict[str, Any] = {"name": model} rates = MODEL_RATES_PER_M.get(model) if rates: @@ -40,33 +66,50 @@ def _model_entry(model: str) -> dict[str, Any]: "output": rates["output"], "cache": {"read": rates["cache_read"], "write": rates["cache_write"]}, } + if provider_family(model).name == OPENAI: + entry["reasoning"] = True + entry["options"] = {"reasoningEffort": REASONING_EFFORT} return entry -def _opencode_config(model: str) -> str: +def _opencode_config(model: str, base_url: str | None = None) -> str: + """opencode.json for one model. `base_url`, when given, pins the endpoint. + + The endpoint belongs under `provider..options.baseURL` -- that options + dict is what opencode forwards to the ai-sdk provider factory (same shape + as amplifier-app-opencode's own writer, cli.py:427-432). A `baseURL` at the + provider root is silently ignored. + + `base_url` is None for the Anthropic path, which instead rewrites + ANTHROPIC_BASE_URL in the run command (see _BASE_URL_NORMALIZE). Leaving + that path's JSON untouched keeps the control arm byte-identical to every + run recorded before this adapter learned about a second family. + """ + family = provider_family(model) + prefix = family.opencode_provider_id + provider_block: dict[str, Any] = {"npm": family.opencode_npm} + if base_url is not None: + provider_block["options"] = {"baseURL": base_url} + provider_block["models"] = {model: _model_entry(model)} return json.dumps( { "$schema": "https://opencode.ai/config.json", - "model": f"anthropic/{model}", + "model": f"{prefix}/{model}", # Pin the SMALL model to the benchmark model too. opencode uses a # separate "small" model for its session-title agent, and its - # default family priority ends at claude-haiku -- a model this - # endpoint may not serve. That request fails with a bare - # `AI_APICallError: Not Found` and kills the process (exit 1) - # before any task work happens; this was deep-swe's single most - # common failure mode for this arm. - "small_model": f"anthropic/{model}", - "provider": { - "anthropic": { - "npm": "@ai-sdk/anthropic", - "models": {model: _model_entry(model)}, - } - }, + # default family priority ends at a cheap model (claude-haiku, + # gpt-*-mini) this endpoint may not serve. That request fails with + # a bare `AI_APICallError: Not Found` and kills the process + # (exit 1) before any task work happens; this was deep-swe's + # single most common failure mode for this arm. + "small_model": f"{prefix}/{model}", + "provider": {prefix: provider_block}, } ) # Normalizes ANTHROPIC_BASE_URL for opencode's ai-sdk provider before the run. +# ANTHROPIC ONLY -- see `opencode_base_url_needs_v1` in jobbench.providers. # # The two clients disagree on what "base URL" means: # * the Anthropic SDK (amplifier-agent, amplifier-foundation) wants the host @@ -78,6 +121,12 @@ def _opencode_config(model: str) -> str: # https://api.anthropic.com/messages, which 404s -- reported as a bare # `Error: Not Found` with no mention of a URL, which looks like a model or # auth problem instead of what it is. +# +# OPENAI_BASE_URL is already the full API root (it ends in /v1), so the OpenAI +# path must NOT run this -- a second /v1 would 404 the same way. The `case` +# below happens to be idempotent, but the OpenAI path skips it outright and +# pins its endpoint in opencode.json instead, which is inspectable after the +# fact in the captured config. _BASE_URL_NORMALIZE = ( 'base="${ANTHROPIC_BASE_URL%/}"; ' 'case "$base" in */v1) ;; *) base="$base/v1" ;; esac; ' @@ -106,12 +155,30 @@ def __init__(self) -> None: async def configure(self, dtu: DTU, *, model: str) -> None: """Write opencode.json at TRIAL time: model, small_model, cost table. - No secret here -- opencode reads ANTHROPIC_API_KEY straight from the - container environment the launch profile's passthrough.services - already populated. + No secret here -- opencode reads the family's API key env var + (ANTHROPIC_API_KEY / OPENAI_API_KEY) straight from the container + environment the launch profile's passthrough.services already + populated. The heredoc below stays QUOTED precisely so nothing in + this JSON can expand against that environment. + + The base URL is not a secret, so the OpenAI path resolves it + host-side and writes it into the config, which also makes the + endpoint under test visible in the captured artifact. Required, not + defaulted: the host must already have it set for passthrough to + forward it at all, and defaulting to api.openai.com could silently + benchmark a different backend. """ self._model = model - config_json = _opencode_config(model) + family = provider_family(model) + base_url: str | None = None + if family.name == OPENAI: + base_url = os.environ.get(family.base_url_env) + if not base_url: + raise AdapterError( + f"opencode-vanilla configure failed: {family.base_url_env} is not set " + f"on the host, so model {model!r} has no endpoint to target" + ) + config_json = _opencode_config(model, base_url) script = ( 'mkdir -p "$HOME/.config/opencode" && ' f"cat > \"{OPENCODE_CONFIG_PATH}\" <<'{_HEREDOC_MARKER}'\n" @@ -128,18 +195,22 @@ async def configure(self, dtu: DTU, *, model: str) -> None: def command(self) -> list[str]: """argv equivalent of: - cd /workspace && opencode run --model anthropic/ --auto \\ + cd /workspace && opencode run --model / --auto \\ "$(cat /workspace/prompt.txt)" - preceded by the ANTHROPIC_BASE_URL normalization above, scoped to this - one exec (there is no persistent env to poison for a later step). + On the Anthropic path this is preceded by the ANTHROPIC_BASE_URL + normalization above, scoped to this one exec (there is no persistent + env to poison for a later step). The OpenAI path has nothing to + normalize -- its endpoint is already pinned in opencode.json. """ if self._model is None: raise AdapterError("opencode-vanilla command() called before configure()") + family = provider_family(self._model) + prefix = f"{_BASE_URL_NORMALIZE}; " if family.opencode_base_url_needs_v1 else "" script = ( - f"{_BASE_URL_NORMALIZE}; " + f"{prefix}" "cd /workspace && " - f"opencode run --model anthropic/{self._model} --auto " + f"opencode run --model {family.opencode_provider_id}/{self._model} --auto " '"$(cat /workspace/prompt.txt)"' ) return ["bash", "-c", script] diff --git a/.amplifier/evaluation/jobbench/src/jobbench/metrics.py b/.amplifier/evaluation/jobbench/src/jobbench/metrics.py index 506338b7..2bbd88d4 100644 --- a/.amplifier/evaluation/jobbench/src/jobbench/metrics.py +++ b/.amplifier/evaluation/jobbench/src/jobbench/metrics.py @@ -226,18 +226,32 @@ def as_dict(self) -> dict[str, Any]: #: USD per 1M tokens, keyed by model id. THE single rate card for this harness. #: -#: Mirrors `_RATES` in amplifier-module-provider-anthropic/_cost.py, which is -#: what stamps `cost_usd` into the amplifier arms' events.jsonl. Arms are only -#: comparable if every dollar figure comes from the same card, so this is also -#: what the opencode arm's cost is RECOMPUTED with -- see -#: `compute_cost_from_tokens` and the WHY in `parse_opencode_db`. +#: Mirrors the upstream provider cost tables that stamp `cost_usd` into the +#: amplifier arms' events.jsonl -- now TWO of them, one per provider family: +#: `_RATES` in amplifier-module-provider-anthropic/_cost.py for the claude-* +#: rows, and the short-context rates in amplifier-module-provider-openai/ +#: _cost.py:102-107 for the gpt-* rows. Arms are only comparable if every +#: dollar figure comes from the same card, so this is also what the opencode +#: arm's cost is RECOMPUTED with -- see `compute_cost_from_tokens` and the WHY +#: in `parse_opencode_db`. +#: +#: All four keys are indexed unconditionally by `compute_cost_from_tokens`, so +#: a row missing one is a KeyError, not a silent zero. #: #: `opencode_vanilla.py` imports this to populate the model's `cost` block in #: opencode.json. One definition, one home. +#: +#: CAVEAT (gpt-5.6-terra): upstream re-rates that model above 272K input +#: tokens; this flat card cannot express a threshold, so it always applies the +#: short-context rate. The amplifier arms are unaffected -- their cost comes +#: from provider events, computed upstream with the real tiering. Only the +#: opencode arm is recomputed here, so ITS terra cost is a FLOOR, understated +#: for any session that crosses the threshold. MODEL_RATES_PER_M: dict[str, dict[str, float]] = { "claude-sonnet-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75}, "claude-sonnet-4-5": {"input": 3.00, "output": 15.00, "cache_read": 0.30, "cache_write": 3.75}, "claude-opus-5": {"input": 5.00, "output": 25.00, "cache_read": 0.50, "cache_write": 6.25}, + "gpt-5.6-terra": {"input": 2.50, "output": 15.00, "cache_read": 0.25, "cache_write": 3.125}, } diff --git a/.amplifier/evaluation/jobbench/src/jobbench/providers.py b/.amplifier/evaluation/jobbench/src/jobbench/providers.py new file mode 100644 index 00000000..ff7c1ab7 --- /dev/null +++ b/.amplifier/evaluation/jobbench/src/jobbench/providers.py @@ -0,0 +1,139 @@ +"""Which provider family a model under test belongs to, and its constants. + +Every arm has to answer the same three questions for whatever `--model` it was +handed: which env vars carry the credential and endpoint, which provider module +to configure, and what opencode calls that provider. This module is the single +place those answers live, so the adapters branch on one derived value instead +of each re-deriving it from the model string. + +Deliberately not a plugin framework: a third family is a third entry in +`_FAMILIES` plus whatever genuinely differs at the two or three call sites. + +The model -> family rule is a prefix test (`gpt-` is OpenAI, everything else is +Anthropic) rather than a lookup table, so a newly released model id works +without editing this file. The one table that DOES need a per-model entry is +`metrics.MODEL_RATES_PER_M`, and only for the opencode arm's cost recompute. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +ANTHROPIC = "anthropic" +OPENAI = "openai" + +#: Benchmark-wide reasoning-effort pin for the OpenAI family. Changing this one +#: value re-pins all three arms (amplifier-agent, amplifier-foundation, +#: opencode-vanilla) together; it is deliberately the only place the literal +#: appears, so the arms cannot drift apart into an invalid comparison. +#: +#: Accepted by provider-openai (validated at mount) as one of: none, minimal, +#: low, medium, high, xhigh, max. opencode spells the same knob +#: `provider..models..options.reasoningEffort` and otherwise applies +#: its own built-in default of "medium" to any model id containing "gpt-5"; +#: this pin overrides exactly that. +#: +#: ANTHROPIC IS NOT PINNED. That family controls reasoning through a thinking +#: token budget, a different mechanism with a different unit, and pinning it is +#: out of scope -- so nothing on the Anthropic path reads this constant. +REASONING_EFFORT = "high" + + +@dataclass(frozen=True) +class ProviderFamily: + """Everything the three in-scope adapters need for one provider family.""" + + #: Family id, ``anthropic`` or ``openai``. Adapters branch on this only + #: where the SHAPE of the config differs, not merely a value. + name: str + + #: Env vars the launch profile's `passthrough.services` forwards into the + #: container. Nothing here reads their VALUES -- only the names, so the + #: secret stays out of our argv and logs. + api_key_env: str + base_url_env: str + + #: Fallback base URL, used only where the existing config already had one. + #: None means "no default, the env var is required" -- the OpenAI endpoint + #: under test is not the public default, so silently falling back to it + #: would benchmark a different backend. + default_base_url: str | None + + #: amplifier-agent host-config `provider.module` short name. Valid values + #: come from amplifier-agent/src/amplifier_agent_cli/provider_sources.py. + agent_module: str + + #: amplifier-foundation settings.yaml provider module + its source, same + #: pairing as provider_sources.py's PROVIDER_SOURCES table. + foundation_module: str + foundation_source: str + + #: opencode provider id. Also the `/` prefix opencode wants for + #: `--model`, `model`, and `small_model`. + opencode_provider_id: str + + #: ai-sdk package opencode loads for this provider. + opencode_npm: str + + #: True when the passthrough base URL is an SDK-style host root that + #: opencode's ai-sdk provider needs a trailing `/v1` appended to. The + #: Anthropic SDK appends /v1 itself so its passthrough value lacks it; + #: OPENAI_BASE_URL already carries it and must not get a second one. + opencode_base_url_needs_v1: bool + + +_FAMILIES: dict[str, ProviderFamily] = { + ANTHROPIC: ProviderFamily( + name=ANTHROPIC, + api_key_env="ANTHROPIC_API_KEY", + base_url_env="ANTHROPIC_BASE_URL", + default_base_url="https://api.anthropic.com", + agent_module="anthropic", + foundation_module="provider-anthropic", + foundation_source=( + "git+https://github.com/microsoft/amplifier-module-provider-anthropic@main" + ), + opencode_provider_id="anthropic", + opencode_npm="@ai-sdk/anthropic", + opencode_base_url_needs_v1=True, + ), + OPENAI: ProviderFamily( + name=OPENAI, + api_key_env="OPENAI_API_KEY", + base_url_env="OPENAI_BASE_URL", + default_base_url=None, + agent_module="openai", + foundation_module="provider-openai", + foundation_source=( + "git+https://github.com/microsoft/amplifier-module-provider-openai@main" + ), + opencode_provider_id="openai", + opencode_npm="@ai-sdk/openai", + opencode_base_url_needs_v1=False, + ), +} + + +def provider_family(model: str) -> ProviderFamily: + """Map a model id to its provider family. + + ``gpt-*`` is OpenAI; everything else is Anthropic, which keeps every + pre-existing model id (claude-sonnet-5, claude-opus-5, ...) on exactly the + path it was on before this function existed. + + Examples: + >>> provider_family("claude-sonnet-5").name + 'anthropic' + >>> provider_family("gpt-5.6-terra").name + 'openai' + """ + return _FAMILIES[OPENAI] if model.startswith("gpt-") else _FAMILIES[ANTHROPIC] + + +__all__ = [ + "ANTHROPIC", + "OPENAI", + "REASONING_EFFORT", + "ProviderFamily", + "provider_family", +]