Fix Luna LM Checklist response parsing

evaluator/scores.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 import importlib
+from collections.abc import Mapping
+from copy import copy
 from typing import Any
 
 _LUNA_JUDGE_MODEL = "gpt-56-reasoning-nano-luna"
+_LUNA_RESPONSE_FORMAT = {"type": "json_object"}
 
 
 def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -18,6 +21,112 @@ def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[
     ]
 
 
+def _as_mapping(value: object) -> Mapping[str, Any] | None:
+    if isinstance(value, Mapping):
+        return value
+    for method_name in ("model_dump", "dict"):
+        method = getattr(value, method_name, None)
+        if callable(method):
+            mapped = method()
+            if isinstance(mapped, Mapping):
+                return mapped
+    return None
+
+
+def _luna_output_text(response: object) -> tuple[Mapping[str, Any], str]:
+    response_data = _as_mapping(response)
+    if response_data is None:
+        raise TypeError(f"Luna Responses API returned unsupported response type {type(response).__name__}")
+
+    text_parts: list[str] = []
+    output = response_data.get("output")
+    if isinstance(output, list):
+        for raw_item in output:
+            item = _as_mapping(raw_item)
+            if item is None:
+                continue
+            item_type = item.get("type")
+            if item_type == "message" and item.get("role") == "assistant":
+                content = item.get("content")
+                if not isinstance(content, list):
+                    continue
+                for raw_part in content:
+                    part = _as_mapping(raw_part)
+                    if part is None or part.get("type") not in ("output_text", "text"):
+                        continue
+                    text = part.get("text")
+                    if isinstance(text, str):
+                        text_parts.append(text)
+            elif item_type in ("output_text", "text"):
+                text = item.get("text") or item.get("content")
+                if isinstance(text, str):
+                    text_parts.append(text)
+
+    if not text_parts:
+        raise ValueError("Luna Responses API response contained no assistant output_text content")
+    return response_data, "".join(text_parts)
+
+
+def _luna_chat_completion(response: object) -> dict[str, Any]:
+    response_data, content = _luna_output_text(response)
+    return {
+        "id": response_data.get("id"),
+        "object": "chat.completion",
+        "created": response_data.get("created_at", response_data.get("created")),
+        "model": response_data.get("model"),
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": content},
+                "finish_reason": response_data.get("stop_reason", "stop"),
+            }
+        ],
+    }
+
+
+def _luna_responses_params(request: dict[str, object]) -> dict[str, object]:
+    supported = {
+        "max_tokens",
+        "messages",
+        "model",
+        "reasoning_effort",
+        "response_format",
+        "span_info",
+    }
+    unsupported = sorted(set(request) - supported)
+    if unsupported:
+        raise ValueError(f"Unsupported Luna LM Checklist request arguments: {', '.join(unsupported)}")
+    if request.get("model") != _LUNA_JUDGE_MODEL:
+        raise ValueError("Luna Responses adapter received a non-Luna model")
+
+    params = {
+        "model": request["model"],
+        "input": request["messages"],
+    }
+    if (max_tokens := request.get("max_tokens")) is not None:
+        params["max_output_tokens"] = max_tokens
+    if (response_format := request.get("response_format")) is not None:
+        if response_format != _LUNA_RESPONSE_FORMAT:
+            raise ValueError(f"Unsupported Luna LM Checklist response format: {response_format!r}")
+        params["text"] = {"format": dict(_LUNA_RESPONSE_FORMAT)}
+    if (reasoning_effort := request.get("reasoning_effort")) is not None:
+        params["reasoning"] = {"effort": reasoning_effort}
+    return params
+
+
+def _luna_responses_client(client: object) -> object:
+    oai = importlib.import_module("autoevals.oai")
+    resolved_client = oai.prepare_openai(client=client, is_async=False)
+    adapted_client = copy(resolved_client)
+
+    def complete(**request: object) -> dict[str, Any]:
+        response = resolved_client.openai.responses.create(**_luna_responses_params(request))
+        return _luna_chat_completion(response)
+
+    adapted_client.complete = complete
+    return adapted_client
+
+
 class LmChecklist:
     def __new__(cls) -> object:
         # bc-eval loads a custom class with this name before its built-in scorer.
@@ -33,6 +142,7 @@ def responses_compatible_request_args(output: object, expected: object = None, *
                 request["messages"] = _responses_compatible_messages(request["messages"])
             if model == _LUNA_JUDGE_MODEL:
                 request.pop("temperature", None)
+                request["client"] = _luna_responses_client(request.get("client"))
             return request
 
         scorer._request_args = responses_compatible_request_args