From 55a8f41b8c9d0f9fa1078e2f3a2300dedd7caf92 Mon Sep 17 00:00:00 2001 From: Rui Martins Date: Wed, 7 Oct 2026 13:14:11 +0200 Subject: [PATCH] Fix Luna LM Checklist response parsing Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- evaluator/scores.py | 110 ++++++++++++++++ tests/test_bceval_scores.py | 252 +++++++++++++++++++++++++++++++++--- 2 files changed, 343 insertions(+), 19 deletions(-) diff --git a/evaluator/scores.py b/evaluator/scores.py index 7679f2580..3c9959213 100644 --- a/evaluator/scores.py +++ b/evaluator/scores.py @@ -1,9 +1,12 @@ from __future__ import annotations import importlib +from collections.abc import Mapping +from copy import copy from typing import Any _LUNA_JUDGE_MODEL = "gpt-56-reasoning-nano-luna" +_LUNA_RESPONSE_FORMAT = {"type": "json_object"} def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -18,6 +21,112 @@ def _responses_compatible_messages(messages: list[dict[str, Any]]) -> list[dict[ ] +def _as_mapping(value: object) -> Mapping[str, Any] | None: + if isinstance(value, Mapping): + return value + for method_name in ("model_dump", "dict"): + method = getattr(value, method_name, None) + if callable(method): + mapped = method() + if isinstance(mapped, Mapping): + return mapped + return None + + +def _luna_output_text(response: object) -> tuple[Mapping[str, Any], str]: + response_data = _as_mapping(response) + if response_data is None: + raise TypeError(f"Luna Responses API returned unsupported response type {type(response).__name__}") + + text_parts: list[str] = [] + output = response_data.get("output") + if isinstance(output, list): + for raw_item in output: + item = _as_mapping(raw_item) + if item is None: + continue + item_type = item.get("type") + if item_type == "message" and item.get("role") == "assistant": + content = item.get("content") + if not isinstance(content, list): + continue + for raw_part in content: + part = _as_mapping(raw_part) + if part is None or part.get("type") not in ("output_text", "text"): + continue + text = part.get("text") + if isinstance(text, str): + text_parts.append(text) + elif item_type in ("output_text", "text"): + text = item.get("text") or item.get("content") + if isinstance(text, str): + text_parts.append(text) + + if not text_parts: + raise ValueError("Luna Responses API response contained no assistant output_text content") + return response_data, "".join(text_parts) + + +def _luna_chat_completion(response: object) -> dict[str, Any]: + response_data, content = _luna_output_text(response) + return { + "id": response_data.get("id"), + "object": "chat.completion", + "created": response_data.get("created_at", response_data.get("created")), + "model": response_data.get("model"), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": response_data.get("stop_reason", "stop"), + } + ], + } + + +def _luna_responses_params(request: dict[str, object]) -> dict[str, object]: + supported = { + "max_tokens", + "messages", + "model", + "reasoning_effort", + "response_format", + "span_info", + } + unsupported = sorted(set(request) - supported) + if unsupported: + raise ValueError(f"Unsupported Luna LM Checklist request arguments: {', '.join(unsupported)}") + if request.get("model") != _LUNA_JUDGE_MODEL: + raise ValueError("Luna Responses adapter received a non-Luna model") + + params = { + "model": request["model"], + "input": request["messages"], + } + if (max_tokens := request.get("max_tokens")) is not None: + params["max_output_tokens"] = max_tokens + if (response_format := request.get("response_format")) is not None: + if response_format != _LUNA_RESPONSE_FORMAT: + raise ValueError(f"Unsupported Luna LM Checklist response format: {response_format!r}") + params["text"] = {"format": dict(_LUNA_RESPONSE_FORMAT)} + if (reasoning_effort := request.get("reasoning_effort")) is not None: + params["reasoning"] = {"effort": reasoning_effort} + return params + + +def _luna_responses_client(client: object) -> object: + oai = importlib.import_module("autoevals.oai") + resolved_client = oai.prepare_openai(client=client, is_async=False) + adapted_client = copy(resolved_client) + + def complete(**request: object) -> dict[str, Any]: + response = resolved_client.openai.responses.create(**_luna_responses_params(request)) + return _luna_chat_completion(response) + + adapted_client.complete = complete + return adapted_client + + class LmChecklist: def __new__(cls) -> object: # bc-eval loads a custom class with this name before its built-in scorer. @@ -33,6 +142,7 @@ def responses_compatible_request_args(output: object, expected: object = None, * request["messages"] = _responses_compatible_messages(request["messages"]) if model == _LUNA_JUDGE_MODEL: request.pop("temperature", None) + request["client"] = _luna_responses_client(request.get("client")) return request scorer._request_args = responses_compatible_request_args diff --git a/tests/test_bceval_scores.py b/tests/test_bceval_scores.py index 1eea27624..3a376e948 100644 --- a/tests/test_bceval_scores.py +++ b/tests/test_bceval_scores.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import sys import types from typing import cast @@ -9,15 +10,104 @@ from evaluator import scores +class _RecordingClient: + def __init__(self, *, responses_result=None, chat_result=None): + self.responses_result = responses_result + self.chat_result = chat_result + self.responses_calls = [] + self.chat_calls = [] + self.openai = types.SimpleNamespace(responses=types.SimpleNamespace(create=self._create_response)) + + def _create_response(self, **kwargs): + self.responses_calls.append(kwargs) + return self.responses_result + + def complete(self, **kwargs): + self.chat_calls.append(kwargs) + if self.chat_result is None: + raise AssertionError("Chat completions should not be used") + return self.chat_result + + +class _ModelResponse: + def __init__(self, data): + self.output = data["output"] + self._data = data + + def model_dump(self): + return self._data + + +def _install_lm_checklist_dependencies(monkeypatch, *, model, client): + class _BuiltInLmChecklist: + def __init__(self): + self.client = client + self.extra_args = {} + + def _request_args(self, output, expected, **kwargs): + return { + "client": self.client, + "model": model, + "temperature": 0, + "max_tokens": 4096, + "response_format": {"type": "json_object"}, + "messages": [ + {"role": "system", "content": "instructions"}, + {"role": "user", "content": "question"}, + {"role": "assistant", "content": output}, + ], + } + + def _process_response(self, response, assertions): + evaluated = json.loads(response["choices"][0]["message"]["content"])["assertions"] + passed = all( + next( + (result["pass"] for result in evaluated if result["text"] == assertion["text"]), + False, + ) + for assertion in assertions + ) + return [ + types.SimpleNamespace(name="pass_rate", score=float(passed)), + types.SimpleNamespace(name="test_passed", score=float(passed)), + ] + + def _run_eval_sync(self, output, expected, **kwargs): + request = self._request_args(output, expected, **kwargs) + request_client = prepare_openai(client=request.pop("client")) + response = request_client.complete(**request) + return self._process_response(response, expected["assertions"]) + + def prepare_openai(*, client=None, is_async=False): + assert not is_async + return client or _RecordingClient() + + bc_eval_module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") + bc_eval_module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, bc_eval_module.__name__, bc_eval_module) + + autoevals_module = types.ModuleType("autoevals.oai") + autoevals_module.prepare_openai = prepare_openai # ty: ignore[unresolved-attribute] + monkeypatch.setitem(sys.modules, autoevals_module.__name__, autoevals_module) + return _BuiltInLmChecklist + + @pytest.mark.parametrize( - ("model", "expected_content", "expects_temperature"), + ("model", "expected_content", "expects_temperature", "adapts_client"), [ ( "gpt-56-reasoning-nano-luna", [{"type": "output_text", "text": "answer"}], False, + True, ), - ("gpt-41-2025-04-14", "answer", True), + ( + "gpt-5-mini", + [{"type": "output_text", "text": "answer"}], + True, + False, + ), + ("gpt-41-2025-04-14", "answer", True, False), ], ) def test_lm_checklist_adapts_only_luna_responses_payload( @@ -25,27 +115,15 @@ def test_lm_checklist_adapts_only_luna_responses_payload( model, expected_content, expects_temperature, + adapts_client, ): - class _BuiltInLmChecklist: - def _request_args(self, output, expected, **kwargs): - return { - "model": model, - "temperature": 0, - "messages": [ - {"role": "system", "content": "instructions"}, - {"role": "user", "content": "question"}, - {"role": "assistant", "content": output}, - ], - } - - module = types.ModuleType("bc_eval.scorers.autoeval.lmchecklist") - module.LmChecklist = _BuiltInLmChecklist # ty: ignore[unresolved-attribute] - monkeypatch.setitem(sys.modules, module.__name__, module) + client = _RecordingClient() + built_in = _install_lm_checklist_dependencies(monkeypatch, model=model, client=client) - scorer = cast(_BuiltInLmChecklist, scores.LmChecklist()) + scorer = cast(built_in, scores.LmChecklist()) request = scorer._request_args("answer", {"assertions": []}) - assert isinstance(scorer, _BuiltInLmChecklist) + assert isinstance(scorer, built_in) assert request["messages"] == [ {"role": "system", "content": "instructions"}, {"role": "user", "content": "question"}, @@ -55,3 +133,139 @@ def _request_args(self, output, expected, **kwargs): }, ] assert ("temperature" in request) is expects_temperature + assert (request["client"] is not client) is adapts_client + + +def test_lm_checklist_parses_nested_luna_response(monkeypatch): + assertion = {"text": "The answer is correct.", "level": "critical"} + judgment = { + "assertions": [ + { + "text": assertion["text"], + "reasoning": "The response matches.", + "pass": True, + } + ] + } + client = _RecordingClient( + responses_result=_ModelResponse( + { + "id": "resp-1", + "object": "response", + "created_at": 123, + "model": "gpt-56-reasoning-nano-luna", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": json.dumps(judgment), + } + ], + } + ], + } + ) + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync( + "answer", + {"assertions": [assertion]}, + input="question", + ) + + assert {score.name: score.score for score in result} == { + "pass_rate": 1.0, + "test_passed": 1.0, + } + assert len(client.responses_calls) == 1 + responses_request = client.responses_calls[0] + assert responses_request["model"] == "gpt-56-reasoning-nano-luna" + assert responses_request["max_output_tokens"] == 4096 + assert responses_request["text"] == {"format": {"type": "json_object"}} + assert responses_request["input"][-1] == { + "role": "assistant", + "content": [{"type": "output_text", "text": "answer"}], + } + assert "max_tokens" not in responses_request + assert "response_format" not in responses_request + assert "temperature" not in responses_request + assert client.chat_calls == [] + + +def test_lm_checklist_delegates_non_luna_requests(monkeypatch): + assertion = {"text": "The answer is correct."} + client = _RecordingClient( + chat_result={ + "choices": [ + { + "message": { + "content": json.dumps( + { + "assertions": [ + { + "text": assertion["text"], + "reasoning": "The response matches.", + "pass": True, + } + ] + } + ) + } + } + ] + } + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-5-mini", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + result = scorer._run_eval_sync("answer", {"assertions": [assertion]}) + + assert {score.name: score.score for score in result}["test_passed"] == 1.0 + assert len(client.chat_calls) == 1 + assert client.responses_calls == [] + + +def test_lm_checklist_rejects_luna_response_without_output_text(monkeypatch): + client = _RecordingClient( + responses_result={ + "id": "resp-1", + "object": "response", + "model": "gpt-56-reasoning-nano-luna", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [], + } + ], + } + ) + built_in = _install_lm_checklist_dependencies( + monkeypatch, + model="gpt-56-reasoning-nano-luna", + client=client, + ) + + scorer = cast(built_in, scores.LmChecklist()) + + with pytest.raises( + ValueError, + match="Luna Responses API response contained no assistant output_text content", + ): + scorer._run_eval_sync( + "answer", + {"assertions": [{"text": "The answer is correct."}]}, + )