From 567fbcb4c039b22ff3e024786b679f0b2f669476 Mon Sep 17 00:00:00 2001
From: Michael Welsch <metricspace.ai@gmail.com>
Date: Sat, 3 Oct 2026 07:52:06 +0200
Subject: [PATCH] Report unexpected data under null retrieval schema as a
 mismatch

---
 .../evaluators/agent_response_evaluator.py    |  7 ++
 .../test_agent_response_nullable.py           | 86 +++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 tests/core/evaluation/evaluators/test_agent_response_nullable.py

diff --git a/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py
index d0486ad..081aa80 100644
--- a/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py
+++ b/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py
@@ -142,6 +142,13 @@ def _normalized_retrieved_data(
         retrieved_data_raw = (
             (retrieved_data_raw,) if not isinstance(retrieved_data_raw, (list, tuple)) else tuple(retrieved_data_raw)
         )
+        if config.results_schema.get("type") == "null":
+            if strict:
+                raise ValueError("Null results schema expects no retrieved_data")
+            # Preserve unexpected actual data so comparison reports a mismatch.
+            # A null-only schema has no array item type to normalize against.
+            return retrieved_data_raw
+
         _derender_url_fct = partial(context.config.derender_url, sites=context.task.sites, strict=strict)
         # TODO: Maybe call normalize array directly here
         return self.value_normalizer.normalize_array(
diff --git a/tests/core/evaluation/evaluators/test_agent_response_nullable.py b/tests/core/evaluation/evaluators/test_agent_response_nullable.py
new file mode 100644
index 0000000..987180f
--- /dev/null
+++ b/tests/core/evaluation/evaluators/test_agent_response_nullable.py
@@ -0,0 +1,86 @@
+from types import SimpleNamespace
+
+import pytest
+
+from webarena_verified.core.evaluation.evaluators.agent_response_evaluator import AgentResponseEvaluator
+from webarena_verified.types.eval import EvalStatus
+from webarena_verified.types.task import AgentResponseEvaluatorCfg
+
+
+@pytest.fixture
+def evaluator_config():
+    return AgentResponseEvaluatorCfg(
+        expected={"task_type": "RETRIEVE", "status": "NOT_FOUND_ERROR", "retrieved_data": None},
+        results_schema={"type": "null"},
+    )
+
+
+def _evaluate(config, response):
+    # Synthetic context: no fixture, browser, network, or historical response is used.
+    context = SimpleNamespace(
+        task=SimpleNamespace(task_id=0, is_retrieve_task=True, sites=[]),
+        config=SimpleNamespace(derender_url=lambda value, **kwargs: value),
+        agent_response_raw=response,
+    )
+    return AgentResponseEvaluator().evaluate(context=context, config=config)
+
+
+@pytest.mark.parametrize("data", [None, []])
+def test_null_schema_accepts_existing_empty_data_policy(evaluator_config, data):
+    result = _evaluate(
+        evaluator_config,
+        {"task_type": "RETRIEVE", "status": "NOT_FOUND_ERROR", "retrieved_data": data},
+    )
+    assert result.status == EvalStatus.SUCCESS
+    assert result.score == 1.0
+    assert result.error_msg is None
+    assert result.expected["retrieved_data"] is None
+
+
+def test_null_schema_accepts_existing_missing_data_policy(evaluator_config):
+    result = _evaluate(evaluator_config, {"task_type": "RETRIEVE", "status": "NOT_FOUND_ERROR"})
+    assert result.status == EvalStatus.SUCCESS
+    assert result.score == 1.0
+
+
+@pytest.mark.parametrize("status", ["NOT_FOUND_ERROR", "SUCCESS"])
+@pytest.mark.parametrize("data", [["unexpected"], [{"review": "unexpected"}], [None], "unexpected", 0, False])
+def test_null_schema_unexpected_data_is_failure_not_error(evaluator_config, status, data):
+    result = _evaluate(evaluator_config, {"task_type": "RETRIEVE", "status": status, "retrieved_data": data})
+    assert result.status == EvalStatus.FAILURE
+    assert result.score == 0.0
+    assert result.error_msg is None
+    assert result.actual_normalized["retrieved_data"] is not None
+    assert result.expected["retrieved_data"] is None
+    assert any(assertion.assertion_name == "retrieved_data_unexpected" for assertion in result.assertions)
+
+
+def test_null_data_does_not_hide_wrong_status(evaluator_config):
+    result = _evaluate(evaluator_config, {"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": None})
+    assert result.status == EvalStatus.FAILURE
+    assert result.score == 0.0
+    assert result.error_msg is None
+
+
+def test_null_schema_still_rejects_nonempty_expected_data():
+    config = AgentResponseEvaluatorCfg(
+        expected={"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": ["invalid oracle"]},
+        results_schema={"type": "null"},
+    )
+    result = _evaluate(config, {"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": None})
+    assert result.status == EvalStatus.ERROR
+    assert "Null results schema expects no retrieved_data" in result.error_msg
+
+
+@pytest.mark.parametrize(
+    ("data", "expected_status"),
+    [(["example"], EvalStatus.SUCCESS), (["different"], EvalStatus.FAILURE), (None, EvalStatus.FAILURE)],
+)
+def test_array_schema_normalization_is_unchanged(data, expected_status):
+    config = AgentResponseEvaluatorCfg(
+        expected={"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": ["Example"]},
+        results_schema={"type": "array", "items": {"type": "string"}},
+    )
+    result = _evaluate(config, {"task_type": "RETRIEVE", "status": "SUCCESS", "retrieved_data": data})
+    assert result.status == expected_status
+    assert result.error_msg is None