diff --git a/bench/surface_bench/models.py b/bench/surface_bench/models.py index 83e205d..e5404c1 100644 --- a/bench/surface_bench/models.py +++ b/bench/surface_bench/models.py @@ -127,6 +127,83 @@ def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step: ) +# ---- Anthropic tool-use translation --------------------------------------------------------- +# Pure converters between the neutral loop format (agent.run_agent) and the Anthropic wire format, +# kept at module scope so they can be unit-tested without a network call (the riskiest part of any +# provider adapter is the message/tool round-trip). + + +def _anthropic_tools(tools: list[dict]) -> list[dict]: + return [ + {"name": t["name"], "description": t["description"], "input_schema": t["parameters"]} + for t in tools + ] + + +def _anthropic_blocks_from_step(step: Step) -> list[dict]: + # Fallback reconstruction when a Step has no provider_msg (e.g. a mock); the real adapter always + # stores provider_msg, so this just keeps history valid for non-Anthropic-authored turns. + blocks: list[dict] = [] + if step.text: + blocks.append({"type": "text", "text": step.text}) + for tc in step.tool_calls: + blocks.append({"type": "tool_use", "id": tc.id, "name": tc.name, "input": tc.args}) + return blocks + + +def _anthropic_messages(messages: list[dict]) -> list[dict]: + # Anthropic requires alternating roles, so we coalesce consecutive same-role turns — in + # particular a tool_result user-turn followed by a nudge user-turn become one user message. + out: list[dict] = [] + + def push(role: str, blocks: list[dict]) -> None: + if out and out[-1]["role"] == role: + out[-1]["content"].extend(blocks) + else: + out.append({"role": role, "content": list(blocks)}) + + for m in messages: + if m["role"] == "user": + push("user", [{"type": "text", "text": m["content"]}]) + elif m["role"] == "assistant": + step = m["step"] + blocks = step.provider_msg if step.provider_msg is not None else _anthropic_blocks_from_step(step) + push("assistant", blocks) + elif m["role"] == "tool": + push( + "user", + [ + {"type": "tool_result", "tool_use_id": r["id"], "content": r["content"]} + for r in m["results"] + ], + ) + return out + + +def _step_from_anthropic(resp) -> Step: + text = "" + calls: list[ToolCall] = [] + provider: list[dict] = [] + for b in resp.content: + btype = getattr(b, "type", None) + if btype == "text": + text += b.text + provider.append({"type": "text", "text": b.text}) + elif btype == "tool_use": + args = dict(b.input) + calls.append(ToolCall(id=b.id, name=b.name, args=args)) + provider.append({"type": "tool_use", "id": b.id, "name": b.name, "input": args}) + u = resp.usage + return Step( + text=text, + tool_calls=calls, + input_tokens=getattr(u, "input_tokens", 0), + output_tokens=getattr(u, "output_tokens", 0), + stop_reason=getattr(resp, "stop_reason", "") or "", + provider_msg=provider, + ) + + class AnthropicModel: def __init__(self, name: str, model_id: str, temperature: float, max_tokens: int): try: @@ -160,6 +237,17 @@ def complete(self, system: str, user: str) -> Completion: raw_usage=u.model_dump() if hasattr(u, "model_dump") else {}, ) + def step(self, system: str, messages: list[dict], tools: list[dict]) -> Step: + resp = self._client.messages.create( + model=self.model_id, + system=system, + max_tokens=self.max_tokens, + temperature=self.temperature, + tools=_anthropic_tools(tools), + messages=_anthropic_messages(messages), + ) + return _step_from_anthropic(resp) + def build_model( name: str, spec: dict, *, temperature: float, max_tokens: int, mode: str = "single" diff --git a/bench/tests/test_anthropic_adapter.py b/bench/tests/test_anthropic_adapter.py new file mode 100644 index 0000000..e049281 --- /dev/null +++ b/bench/tests/test_anthropic_adapter.py @@ -0,0 +1,72 @@ +"""Offline tests for the Anthropic tool-use translation (no network — pure converters + a fake +response). The live round-trip is covered by a manual smoke, not the suite.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from surface_bench.models import ( + Step, + ToolCall, + _anthropic_messages, + _anthropic_tools, + _step_from_anthropic, +) +from surface_bench.tools_runtime import TOOL_SPECS + + +def test_tools_translate_to_input_schema() -> None: + out = _anthropic_tools(TOOL_SPECS) + assert {t["name"] for t in out} == {"list_dir", "read_file", "grep", "final_answer"} + for spec, t in zip(TOOL_SPECS, out): + assert t["input_schema"] == spec["parameters"] # Anthropic's field name + assert set(t) == {"name", "description", "input_schema"} + + +def test_messages_roundtrip_and_coalescing() -> None: + # Assistant turn produced by the adapter carries provider_msg (echoed verbatim). + asst = Step( + tool_calls=[ToolCall(id="t1", name="read_file", args={"path": "code/x.py"})], + provider_msg=[{"type": "tool_use", "id": "t1", "name": "read_file", "input": {"path": "code/x.py"}}], + ) + messages = [ + {"role": "user", "content": "do the task"}, + {"role": "assistant", "step": asst}, + {"role": "tool", "results": [{"id": "t1", "content": "WINDOW_LIMIT = 10"}]}, + {"role": "user", "content": "answer now"}, # the forced-answer nudge + ] + out = _anthropic_messages(messages) + + # roles must alternate: user, assistant, user (the tool-result turn + nudge are coalesced) + assert [m["role"] for m in out] == ["user", "assistant", "user"] + assert out[1]["content"] == asst.provider_msg + last = out[2]["content"] + assert last[0]["type"] == "tool_result" and last[0]["tool_use_id"] == "t1" + assert last[1] == {"type": "text", "text": "answer now"} + + +def test_messages_reconstruct_when_no_provider_msg() -> None: + # A Step without provider_msg (e.g. a mock) is reconstructed into valid blocks. + asst = Step(text="thinking", tool_calls=[ToolCall(id="t1", name="grep", args={"pattern": "x"})]) + out = _anthropic_messages([{"role": "assistant", "step": asst}]) + blocks = out[0]["content"] + assert {"type": "text", "text": "thinking"} in blocks + assert {"type": "tool_use", "id": "t1", "name": "grep", "input": {"pattern": "x"}} in blocks + + +def test_step_parsed_from_response() -> None: + resp = SimpleNamespace( + content=[ + SimpleNamespace(type="text", text="let me check"), + SimpleNamespace(type="tool_use", id="t9", name="read_file", input={"path": "code/x.py"}), + ], + usage=SimpleNamespace(input_tokens=120, output_tokens=18), + stop_reason="tool_use", + ) + step = _step_from_anthropic(resp) + assert step.text == "let me check" + assert step.tool_calls == [ToolCall(id="t9", name="read_file", args={"path": "code/x.py"})] + assert (step.input_tokens, step.output_tokens) == (120, 18) + assert step.stop_reason == "tool_use" + # provider_msg must round-trip back into a valid assistant message + assert _anthropic_messages([{"role": "assistant", "step": step}])[0]["content"] == step.provider_msg