feat(bench): OpenAI + Gemini tool-use adapters (#107)

bench/config.toml

@@ -33,3 +33,18 @@ provider = "anthropic"
 model_id = "claude-opus-4-8"
 input_per_mtok = 5.00
 output_per_mtok = 25.00
+
+# Cross-provider adapters (#107) for the multi-turn generalization sweep. Need the [providers] extra
+# (`uv sync --extra providers`) + OPENAI_API_KEY / GEMINI_API_KEY. TODO at first smoke: pin the exact
+# model ids and per-Mtok pricing below (placeholders).
+[models.gpt]
+provider = "openai"
+model_id = "gpt-5.1"          # TODO verify exact id at smoke time
+input_per_mtok = 0.00
+output_per_mtok = 0.00
+
+[models.gemini]
+provider = "gemini"
+model_id = "gemini-2.5-pro"   # TODO verify exact id at smoke time
+input_per_mtok = 0.00
+output_per_mtok = 0.00

bench/pyproject.toml

@@ -10,6 +10,8 @@ dependencies = [
 [project.optional-dependencies]
 plots = ["matplotlib>=3.8"]
 dev = ["pytest>=8"]
+# Cross-provider adapters (#107); not needed for the Anthropic-only or offline paths.
+providers = ["openai>=1.0", "google-genai>=0.3"]
 
 [tool.setuptools.packages.find]
 include = ["surface_bench*"]

bench/surface_bench/models.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 from dataclasses import dataclass, field
 from typing import Any, Protocol
@@ -249,6 +250,198 @@ def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step:
         return _step_from_anthropic(resp)
 
 
+# ---- OpenAI tool-use translation ------------------------------------------------------------
+# Same pattern as the Anthropic block: pure converters between the neutral loop format and the
+# OpenAI Chat Completions wire format, unit-tested without a network call. NOTE for smoke time:
+# newer GPT models may need `max_completion_tokens` instead of `max_tokens` and may reject a custom
+# `temperature` — adjust in OpenAIModel.step when the exact model id is pinned.
+
+
+def _json_args(raw: str) -> dict:
+    try:
+        return json.loads(raw) if raw else {}
+    except json.JSONDecodeError:
+        return {}  # a malformed tool call becomes empty args; the loop feeds back an error and recovers
+
+
+def _openai_tools(tools: list[dict]) -> list[dict]:
+    return [
+        {
+            "type": "function",
+            "function": {
+                "name": t["name"],
+                "description": t["description"],
+                "parameters": t["parameters"],
+            },
+        }
+        for t in tools
+    ]
+
+
+def _openai_messages(system: str, messages: list[dict]) -> list[dict]:
+    out: list[dict] = [{"role": "system", "content": system}]
+    for m in messages:
+        if m["role"] == "user":
+            out.append({"role": "user", "content": m["content"]})
+        elif m["role"] == "assistant":
+            step = m["step"]
+            msg: dict = {"role": "assistant", "content": step.text or None}
+            if step.tool_calls:
+                msg["tool_calls"] = [
+                    {
+                        "id": tc.id,
+                        "type": "function",
+                        "function": {"name": tc.name, "arguments": json.dumps(tc.args)},
+                    }
+                    for tc in step.tool_calls
+                ]
+            out.append(msg)
+        elif m["role"] == "tool":
+            for r in m["results"]:
+                out.append({"role": "tool", "tool_call_id": r["id"], "content": r["content"]})
+    return out
+
+
+def _step_from_openai(resp) -> Step:
+    choice = resp.choices[0]
+    msg = choice.message
+    calls = [
+        ToolCall(id=tc.id, name=tc.function.name, args=_json_args(tc.function.arguments))
+        for tc in (msg.tool_calls or [])
+    ]
+    u = getattr(resp, "usage", None)
+    return Step(
+        text=msg.content or "",
+        tool_calls=calls,
+        input_tokens=getattr(u, "prompt_tokens", 0) if u else 0,
+        output_tokens=getattr(u, "completion_tokens", 0) if u else 0,
+        stop_reason=getattr(choice, "finish_reason", "") or "",
+    )
+
+
+class OpenAIModel:
+    def __init__(self, name: str, model_id: str, temperature: float, max_tokens: int):
+        try:
+            from openai import OpenAI
+        except ImportError as e:  # pragma: no cover
+            raise SystemExit("pip install openai  (uv sync --extra providers)") from e
+        if not os.environ.get("OPENAI_API_KEY"):
+            raise SystemExit("OPENAI_API_KEY is not set")
+        self.name = name
+        self.model_id = model_id
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self._client = OpenAI(timeout=120.0, max_retries=4)
+
+    def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step:
+        resp = self._client.chat.completions.create(
+            model=self.model_id,
+            messages=_openai_messages(system, messages),
+            tools=_openai_tools(tools),
+            tool_choice="auto",
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+        )
+        return _step_from_openai(resp)
+
+
+# ---- Gemini tool-use translation ------------------------------------------------------------
+# google-genai uses "model"/"user" roles, a system_instruction, and function_call/function_response
+# parts (matched by function *name*, not an id). The converters emit plain dicts (the SDK coerces
+# them), so they stay import-free and testable. NOTE for smoke time: verify the SDK accepts dict-form
+# tools/contents for the pinned version; if not, wrap with google.genai.types in GeminiModel.step.
+
+
+def _gemini_tools(tools: list[dict]) -> list[dict]:
+    return [
+        {
+            "function_declarations": [
+                {"name": t["name"], "description": t["description"], "parameters": t["parameters"]}
+                for t in tools
+            ]
+        }
+    ]
+
+
+def _gemini_contents(messages: list[dict]) -> list[dict]:
+    contents: list[dict] = []
+    id_to_name: dict[str, str] = {}  # Gemini keys function_response by name, not id
+    for m in messages:
+        if m["role"] == "user":
+            contents.append({"role": "user", "parts": [{"text": m["content"]}]})
+        elif m["role"] == "assistant":
+            step = m["step"]
+            parts: list[dict] = []
+            if step.text:
+                parts.append({"text": step.text})
+            for tc in step.tool_calls:
+                parts.append({"function_call": {"name": tc.name, "args": tc.args}})
+                id_to_name[tc.id] = tc.name
+            contents.append({"role": "model", "parts": parts})
+        elif m["role"] == "tool":
+            parts = [
+                {
+                    "function_response": {
+                        "name": id_to_name.get(r["id"], r.get("name", "")),
+                        "response": {"result": r["content"]},
+                    }
+                }
+                for r in m["results"]
+            ]
+            contents.append({"role": "user", "parts": parts})
+    return contents
+
+
+def _step_from_gemini(resp) -> Step:
+    cand = resp.candidates[0]
+    text = ""
+    calls: list[ToolCall] = []
+    for i, part in enumerate(cand.content.parts):
+        fc = getattr(part, "function_call", None)
+        if fc is not None:
+            args = dict(fc.args) if getattr(fc, "args", None) else {}
+            calls.append(ToolCall(id=f"{fc.name}-{i}", name=fc.name, args=args))
+        elif getattr(part, "text", None):
+            text += part.text
+    u = getattr(resp, "usage_metadata", None)
+    return Step(
+        text=text,
+        tool_calls=calls,
+        input_tokens=getattr(u, "prompt_token_count", 0) if u else 0,
+        output_tokens=getattr(u, "candidates_token_count", 0) if u else 0,
+        stop_reason=str(getattr(cand, "finish_reason", "") or ""),
+    )
+
+
+class GeminiModel:
+    def __init__(self, name: str, model_id: str, temperature: float, max_tokens: int):
+        try:
+            from google import genai
+        except ImportError as e:  # pragma: no cover
+            raise SystemExit("pip install google-genai  (uv sync --extra providers)") from e
+        if not (os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")):
+            raise SystemExit("GEMINI_API_KEY (or GOOGLE_API_KEY) is not set")
+        self.name = name
+        self.model_id = model_id
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self._client = genai.Client()
+
+    def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step:
+        from google.genai import types
+
+        config = types.GenerateContentConfig(
+            system_instruction=system,
+            tools=_gemini_tools(tools),
+            temperature=self.temperature,
+            max_output_tokens=self.max_tokens,
+        )
+        resp = self._client.models.generate_content(
+            model=self.model_id, contents=_gemini_contents(messages), config=config
+        )
+        return _step_from_gemini(resp)
+
+
 def build_model(
     name: str, spec: dict, *, temperature: float, max_tokens: int, mode: str = "single"
 ) -> Model:
@@ -259,8 +452,9 @@ def build_model(
                 name=name, default=spec.get("default", ""), replies=spec.get("replies")
             )
         return MockModel(name=name, default=spec.get("default", ""), replies=spec.get("replies"))
-    if provider == "anthropic":
-        return AnthropicModel(
+    cls = {"anthropic": AnthropicModel, "openai": OpenAIModel, "gemini": GeminiModel}.get(provider)
+    if cls is not None:
+        return cls(
             name=name,
             model_id=spec["model_id"],
             temperature=spec.get("temperature", temperature),

bench/tests/test_provider_adapters.py

@@ -0,0 +1,139 @@
+"""Offline tests for the OpenAI + Gemini tool-use translation (no network — pure converters and a
+fake response). Live round-trips are smoke-tested manually, not in the suite (#107)."""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+
+from surface_bench.models import (
+    Step,
+    ToolCall,
+    _gemini_contents,
+    _gemini_tools,
+    _openai_messages,
+    _openai_tools,
+    _step_from_gemini,
+    _step_from_openai,
+    build_model,
+)
+from surface_bench.tools_runtime import TOOL_SPECS
+
+
+# ---- OpenAI ---------------------------------------------------------------------------------
+
+
+def test_openai_tools_and_messages() -> None:
+    tools = _openai_tools(TOOL_SPECS)
+    assert all(t["type"] == "function" for t in tools)
+    assert {t["function"]["name"] for t in tools} == {"list_dir", "read_file", "grep", "final_answer"}
+    assert tools[0]["function"]["parameters"] == TOOL_SPECS[0]["parameters"]
+
+    asst = Step(tool_calls=[ToolCall(id="c1", name="read_file", args={"path": "code/x.py"})])
+    messages = [
+        {"role": "user", "content": "task"},
+        {"role": "assistant", "step": asst},
+        {"role": "tool", "results": [{"id": "c1", "content": "WINDOW_LIMIT = 10"}]},
+    ]
+    out = _openai_messages("sys", messages)
+    assert out[0] == {"role": "system", "content": "sys"}
+    assert out[1] == {"role": "user", "content": "task"}
+    tc = out[2]["tool_calls"][0]
+    assert tc["id"] == "c1" and tc["function"]["name"] == "read_file"
+    assert json.loads(tc["function"]["arguments"]) == {"path": "code/x.py"}
+    assert out[3] == {"role": "tool", "tool_call_id": "c1", "content": "WINDOW_LIMIT = 10"}
+
+
+def test_step_from_openai_parses_tool_calls() -> None:
+    resp = SimpleNamespace(
+        choices=[
+            SimpleNamespace(
+                message=SimpleNamespace(
+                    content="checking",
+                    tool_calls=[
+                        SimpleNamespace(
+                            id="c9",
+                            function=SimpleNamespace(
+                                name="read_file", arguments='{"path": "code/x.py"}'
+                            ),
+                        )
+                    ],
+                ),
+                finish_reason="tool_calls",
+            )
+        ],
+        usage=SimpleNamespace(prompt_tokens=100, completion_tokens=12),
+    )
+    step = _step_from_openai(resp)
+    assert step.text == "checking"
+    assert step.tool_calls == [ToolCall(id="c9", name="read_file", args={"path": "code/x.py"})]
+    assert (step.input_tokens, step.output_tokens) == (100, 12)
+    assert step.stop_reason == "tool_calls"
+
+
+def test_step_from_openai_tolerates_no_tool_calls_and_bad_json() -> None:
+    resp = SimpleNamespace(
+        choices=[SimpleNamespace(message=SimpleNamespace(content="final", tool_calls=None), finish_reason="stop")],
+        usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2),
+    )
+    assert _step_from_openai(resp).tool_calls == []
+
+
+# ---- Gemini ---------------------------------------------------------------------------------
+
+
+def test_gemini_tools_and_function_response_name_resolution() -> None:
+    tools = _gemini_tools(TOOL_SPECS)
+    decls = tools[0]["function_declarations"]
+    assert {d["name"] for d in decls} == {"list_dir", "read_file", "grep", "final_answer"}
+
+    asst = Step(tool_calls=[ToolCall(id="g1", name="read_file", args={"path": "code/x.py"})])
+    messages = [
+        {"role": "user", "content": "task"},
+        {"role": "assistant", "step": asst},
+        {"role": "tool", "results": [{"id": "g1", "content": "WINDOW_LIMIT = 10"}]},
+    ]
+    contents = _gemini_contents(messages)
+    assert [c["role"] for c in contents] == ["user", "model", "user"]
+    assert contents[1]["parts"][0]["function_call"] == {"name": "read_file", "args": {"path": "code/x.py"}}
+    # the tool result is keyed by function NAME (resolved from the prior call's id), not the id
+    fr = contents[2]["parts"][0]["function_response"]
+    assert fr["name"] == "read_file"
+    assert fr["response"] == {"result": "WINDOW_LIMIT = 10"}
+
+
+def test_step_from_gemini_parses_parts() -> None:
+    resp = SimpleNamespace(
+        candidates=[
+            SimpleNamespace(
+                content=SimpleNamespace(
+                    parts=[
+                        SimpleNamespace(function_call=None, text="let me look"),
+                        SimpleNamespace(
+                            function_call=SimpleNamespace(name="read_file", args={"path": "code/x.py"}),
+                            text=None,
+                        ),
+                    ]
+                ),
+                finish_reason="STOP",
+            )
+        ],
+        usage_metadata=SimpleNamespace(prompt_token_count=80, candidates_token_count=9),
+    )
+    step = _step_from_gemini(resp)
+    assert step.text == "let me look"
+    assert step.tool_calls[0].name == "read_file" and step.tool_calls[0].args == {"path": "code/x.py"}
+    assert (step.input_tokens, step.output_tokens) == (80, 9)
+    assert step.stop_reason == "STOP"
+
+
+# ---- registry -------------------------------------------------------------------------------
+
+
+def test_build_model_unknown_provider_still_raises() -> None:
+    try:
+        build_model("x", {"provider": "nope"}, temperature=1.0, max_tokens=10)
+    except ValueError as e:
+        assert "unknown provider" in str(e)
+    else:
+        raise AssertionError("expected ValueError")