feat(bench): Anthropic tool-use adapter for the multi-turn loop (#102)

bench/surface_bench/models.py

@@ -127,6 +127,83 @@ def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step:
         )
 
 
+# ---- Anthropic tool-use translation ---------------------------------------------------------
+# Pure converters between the neutral loop format (agent.run_agent) and the Anthropic wire format,
+# kept at module scope so they can be unit-tested without a network call (the riskiest part of any
+# provider adapter is the message/tool round-trip).
+
+
+def _anthropic_tools(tools: list[dict]) -> list[dict]:
+    return [
+        {"name": t["name"], "description": t["description"], "input_schema": t["parameters"]}
+        for t in tools
+    ]
+
+
+def _anthropic_blocks_from_step(step: Step) -> list[dict]:
+    # Fallback reconstruction when a Step has no provider_msg (e.g. a mock); the real adapter always
+    # stores provider_msg, so this just keeps history valid for non-Anthropic-authored turns.
+    blocks: list[dict] = []
+    if step.text:
+        blocks.append({"type": "text", "text": step.text})
+    for tc in step.tool_calls:
+        blocks.append({"type": "tool_use", "id": tc.id, "name": tc.name, "input": tc.args})
+    return blocks
+
+
+def _anthropic_messages(messages: list[dict]) -> list[dict]:
+    # Anthropic requires alternating roles, so we coalesce consecutive same-role turns — in
+    # particular a tool_result user-turn followed by a nudge user-turn become one user message.
+    out: list[dict] = []
+
+    def push(role: str, blocks: list[dict]) -> None:
+        if out and out[-1]["role"] == role:
+            out[-1]["content"].extend(blocks)
+        else:
+            out.append({"role": role, "content": list(blocks)})
+
+    for m in messages:
+        if m["role"] == "user":
+            push("user", [{"type": "text", "text": m["content"]}])
+        elif m["role"] == "assistant":
+            step = m["step"]
+            blocks = step.provider_msg if step.provider_msg is not None else _anthropic_blocks_from_step(step)
+            push("assistant", blocks)
+        elif m["role"] == "tool":
+            push(
+                "user",
+                [
+                    {"type": "tool_result", "tool_use_id": r["id"], "content": r["content"]}
+                    for r in m["results"]
+                ],
+            )
+    return out
+
+
+def _step_from_anthropic(resp) -> Step:
+    text = ""
+    calls: list[ToolCall] = []
+    provider: list[dict] = []
+    for b in resp.content:
+        btype = getattr(b, "type", None)
+        if btype == "text":
+            text += b.text
+            provider.append({"type": "text", "text": b.text})
+        elif btype == "tool_use":
+            args = dict(b.input)
+            calls.append(ToolCall(id=b.id, name=b.name, args=args))
+            provider.append({"type": "tool_use", "id": b.id, "name": b.name, "input": args})
+    u = resp.usage
+    return Step(
+        text=text,
+        tool_calls=calls,
+        input_tokens=getattr(u, "input_tokens", 0),
+        output_tokens=getattr(u, "output_tokens", 0),
+        stop_reason=getattr(resp, "stop_reason", "") or "",
+        provider_msg=provider,
+    )
+
+
 class AnthropicModel:
     def __init__(self, name: str, model_id: str, temperature: float, max_tokens: int):
         try:
@@ -160,6 +237,17 @@ def complete(self, system: str, user: str) -> Completion:
             raw_usage=u.model_dump() if hasattr(u, "model_dump") else {},
         )
 
+    def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step:
+        resp = self._client.messages.create(
+            model=self.model_id,
+            system=system,
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+            tools=_anthropic_tools(tools),
+            messages=_anthropic_messages(messages),
+        )
+        return _step_from_anthropic(resp)
+
 
 def build_model(
     name: str, spec: dict, *, temperature: float, max_tokens: int, mode: str = "single"

bench/tests/test_anthropic_adapter.py

@@ -0,0 +1,72 @@
+"""Offline tests for the Anthropic tool-use translation (no network — pure converters + a fake
+response). The live round-trip is covered by a manual smoke, not the suite."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from surface_bench.models import (
+    Step,
+    ToolCall,
+    _anthropic_messages,
+    _anthropic_tools,
+    _step_from_anthropic,
+)
+from surface_bench.tools_runtime import TOOL_SPECS
+
+
+def test_tools_translate_to_input_schema() -> None:
+    out = _anthropic_tools(TOOL_SPECS)
+    assert {t["name"] for t in out} == {"list_dir", "read_file", "grep", "final_answer"}
+    for spec, t in zip(TOOL_SPECS, out):
+        assert t["input_schema"] == spec["parameters"]  # Anthropic's field name
+        assert set(t) == {"name", "description", "input_schema"}
+
+
+def test_messages_roundtrip_and_coalescing() -> None:
+    # Assistant turn produced by the adapter carries provider_msg (echoed verbatim).
+    asst = Step(
+        tool_calls=[ToolCall(id="t1", name="read_file", args={"path": "code/x.py"})],
+        provider_msg=[{"type": "tool_use", "id": "t1", "name": "read_file", "input": {"path": "code/x.py"}}],
+    )
+    messages = [
+        {"role": "user", "content": "do the task"},
+        {"role": "assistant", "step": asst},
+        {"role": "tool", "results": [{"id": "t1", "content": "WINDOW_LIMIT = 10"}]},
+        {"role": "user", "content": "answer now"},  # the forced-answer nudge
+    ]
+    out = _anthropic_messages(messages)
+
+    # roles must alternate: user, assistant, user  (the tool-result turn + nudge are coalesced)
+    assert [m["role"] for m in out] == ["user", "assistant", "user"]
+    assert out[1]["content"] == asst.provider_msg
+    last = out[2]["content"]
+    assert last[0]["type"] == "tool_result" and last[0]["tool_use_id"] == "t1"
+    assert last[1] == {"type": "text", "text": "answer now"}
+
+
+def test_messages_reconstruct_when_no_provider_msg() -> None:
+    # A Step without provider_msg (e.g. a mock) is reconstructed into valid blocks.
+    asst = Step(text="thinking", tool_calls=[ToolCall(id="t1", name="grep", args={"pattern": "x"})])
+    out = _anthropic_messages([{"role": "assistant", "step": asst}])
+    blocks = out[0]["content"]
+    assert {"type": "text", "text": "thinking"} in blocks
+    assert {"type": "tool_use", "id": "t1", "name": "grep", "input": {"pattern": "x"}} in blocks
+
+
+def test_step_parsed_from_response() -> None:
+    resp = SimpleNamespace(
+        content=[
+            SimpleNamespace(type="text", text="let me check"),
+            SimpleNamespace(type="tool_use", id="t9", name="read_file", input={"path": "code/x.py"}),
+        ],
+        usage=SimpleNamespace(input_tokens=120, output_tokens=18),
+        stop_reason="tool_use",
+    )
+    step = _step_from_anthropic(resp)
+    assert step.text == "let me check"
+    assert step.tool_calls == [ToolCall(id="t9", name="read_file", args={"path": "code/x.py"})]
+    assert (step.input_tokens, step.output_tokens) == (120, 18)
+    assert step.stop_reason == "tool_use"
+    # provider_msg must round-trip back into a valid assistant message
+    assert _anthropic_messages([{"role": "assistant", "step": step}])[0]["content"] == step.provider_msg