Fix final-page event evaluation and five verified task contracts

assets/dataset/webarena-verified.json

@@ -8593,7 +8593,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
-        "expected": {"url": "^__GITLAB__/a11yproject/a11yproject.com/-/issues/.*$"},
+        "expected": {"url": "^__GITLAB__/a11yproject/a11yproject\\.com/-/issues/?$"},
         "ignored_query_params_patterns": [".*"]
       },
       {
@@ -8608,7 +8608,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -12233,13 +12233,13 @@
             "project[name]": "chatgpt_plugin",
             "project[namespace_id]": "2505",
             "project[path]": "chatgpt_plugin",
-            "project[initialize_with_readme]": "0"
+            "project[initialize_with_readme]": [null, "0"]
           },
           "response_status": 302
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -19058,6 +19058,8 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
+        "event_query_params": {"namespace": "sales_order_grid"},
         "expected": {
           "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$",
           "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"},
@@ -19072,7 +19074,7 @@
         "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"]
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19859,15 +19861,15 @@
           "response_status": 302,
           "post_data": {
             "name": "spring sale",
-            "website_ids": [1],
-            "customer_group_ids": [1],
+            "website_ids[0]": "1",
+            "customer_group_ids[0]": "1",
             "simple_action": "by_percent",
             "discount_amount": 20
           }
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19891,15 +19893,15 @@
           "response_status": 302,
           "post_data": {
             "name": "fall discount",
-            "website_ids": [1],
-            "customer_group_ids": [1],
+            "website_ids[0]": "1",
+            "customer_group_ids[0]": "1",
             "simple_action": "cart_fixed",
             "discount_amount": 10
           }
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],

docs/evaluation/network_event_based_evaluation.md

@@ -116,9 +116,21 @@ right page:
 
 ### 4. Event Types and Sequencing
 
-Set `event_type` to `"navigation"` to focus on page loads, or to `"modification"` for form
-submissions. The `last_event_only` flag instructs the evaluator to match the most recent event;
-disabling it means "any matching event is sufficient".
+GET checks on navigation tasks validate the final document navigation by default.
+For a page whose state is loaded through XHR or fetch, set `navigation_only` to `false`.
+Only requests after the final document navigation are then eligible, so leaving or
+reloading the page invalidates earlier requests.
+
+`event_query_params` selects a request stream by exact query values, for example
+`{"namespace": "sales_order_grid"}` when several grids share one endpoint. Keep
+the state being checked, such as `filters[status]`, in `expected.query_params`.
+`last_event_only` defaults to `true` and checks the last request in that stream;
+setting it to `false` accepts any matching request and is unsuitable when the task
+requires the final page state.
+
+Expected scalar alternatives may explicitly include `null`, such as `[null, "0"]`
+for an unchecked checkbox that may be omitted or sent as zero. An empty string
+alternative alone does not permit a missing value.
 
 ## Network Trace Structure
 

src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py

@@ -8,6 +8,7 @@
 from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, cast
+from urllib.parse import parse_qs, urlsplit
 
 from webarena_verified.core.evaluation.data_types import URL
 from webarena_verified.core.utils import logger
@@ -558,6 +559,26 @@ def _extract_event_data(
             response_cookies=response_cookies,
         )
 
+    @staticmethod
+    def _select_navigation_events(
+        events: tuple[NetworkEvent, ...], context: TaskEvalContext, config: NetworkEventEvaluatorCfg
+    ) -> tuple[tuple[NetworkEvent, ...], bool]:
+        if not (context.task.is_navigate_task and config.expected.http_method == "GET"):
+            return events, False
+        navigations = [index for index, event in enumerate(events) if event.is_navigation_event]
+        if not navigations:
+            return (), True
+        last_navigation = navigations[-1]
+        if config.navigation_only:
+            return (events[last_navigation],), True
+        # Earlier XHRs describe a page the agent has left or reloaded.
+        return events[last_navigation + 1 :], False
+
+    @staticmethod
+    def _matches_event_query(event: NetworkEvent, config: NetworkEventEvaluatorCfg) -> bool:
+        query = parse_qs(urlsplit(event.url).query, keep_blank_values=True)
+        return all(query.get(key) == [value] for key, value in (config.event_query_params or {}).items())
+
     def _filter_events_by_criteria(
         self, events: tuple[NetworkEvent, ...], context: TaskEvalContext, config: NetworkEventEvaluatorCfg
     ) -> tuple[NetworkEvent, ...]:
@@ -572,15 +593,9 @@ def _filter_events_by_criteria(
             Filtered list of events matching URL and headers criteria
         """
 
-        if not events:
-            return ()
-
-        # Handle navigate tasks (We only care about the last navigation event)
-        # For cases where we check navigation events via subsequent XHR/fetch requests,
-        # we use the normal filtering logic below.
-        if context.task.is_navigate_task and config.expected.http_method == "GET":
-            last_navigation_event = [e for e in events if e.is_navigation_event]
-            return (last_navigation_event[-1],) if last_navigation_event else ()
+        events, final_navigation = self._select_navigation_events(events, context, config)
+        if final_navigation or not events:
+            return events
 
         matches = []
         try:
@@ -595,6 +610,8 @@ def _filter_events_by_criteria(
             raise
 
         for event in events:
+            if not self._matches_event_query(event, config):
+                continue
             # Check HTTP method
             if config.expected.http_method and event.http_method.lower() != config.expected.http_method.lower():
                 continue

src/webarena_verified/core/evaluation/value_comparator.py

@@ -458,7 +458,14 @@ def _compare_recursive(
             visited.add(actual_id)
 
         # Handle None values
-        if expected is None and actual is None:
+        if actual is None and (
+            expected is None
+            or (
+                isinstance(expected, NormalizedType)
+                and isinstance(expected.raw, (list, tuple))
+                and any(value is None for value in expected.raw)
+            )
+        ):
             return []
 
         if expected is None or actual is None:

src/webarena_verified/types/task.py

@@ -228,6 +228,21 @@ class NetworkEventEvaluatorCfg(BaseEval[NetworkEventSpec]):
     last_event_only: bool = True
     """If True, validate only the last matching event. If False, validate if ANY event matches."""
 
+    navigation_only: bool = True
+    """For GET checks on navigation tasks, validate the final document navigation.
+
+    Set False for XHR/fetch checks on the final page. Those checks consider only
+    events after the final document navigation and still honor last_event_only.
+    Other task types and HTTP methods are unaffected.
+    """
+
+    event_query_params: dict[str, str] | None = None
+    """Exact query values that identify an event stream before selecting its last event.
+
+    Use stable identifiers such as a grid namespace, not the state being verified.
+    Expected query_params still validate the selected event's complete state.
+    """
+
     ignored_query_params: tuple[str, ...] | None = None
     """Query parameter names to ignore during comparison (literal matching)."""
 

tests/api/test_corrected_task_contracts.py

@@ -0,0 +1,117 @@
+"""Exercise corrected contracts through the public HAR evaluation API."""
+
+import json
+from pathlib import Path
+from urllib.parse import urlencode
+
+import pytest
+
+from webarena_verified.api import WebArenaVerified
+from webarena_verified.types.config import WebArenaVerifiedConfig
+from webarena_verified.types.eval import EvalStatus
+
+
+def _event(url, *, method="GET", form=None, referer=None, navigation=False):
+    headers = [{"name": "Referer", "value": referer}] if referer else []
+    if navigation:
+        headers += [
+            {"name": "Accept", "value": "text/html"},
+            {"name": "Sec-Fetch-Dest", "value": "document"},
+            {"name": "Sec-Fetch-Mode", "value": "navigate"},
+            {"name": "Sec-Fetch-User", "value": "?1"},
+        ]
+    request = {"method": method, "url": url, "headers": headers, "cookies": [], "queryString": []}
+    if form is not None:
+        request["postData"] = {"mimeType": "application/x-www-form-urlencoded", "text": urlencode(form)}
+    return {
+        "startedDateTime": "2026-01-01T00:00:00.000Z",
+        "time": 1,
+        "request": request,
+        "response": {
+            "status": 302 if form is not None else 200,
+            "headers": [],
+            "cookies": [],
+            "content": {"size": 0, "mimeType": "text/html", "text": ""},
+            "redirectURL": "",
+        },
+        "cache": {},
+        "timings": {"send": 0, "wait": 1, "receive": 0},
+    }
+
+
+def _evaluate(tmp_path, task, entries, task_type):
+    path = tmp_path / "network.har"
+    path.write_text(
+        json.dumps({"log": {"version": "1.2", "creator": {"name": "pytest", "version": "1"}, "entries": entries}})
+    )
+    evaluator = WebArenaVerified(
+        config=WebArenaVerifiedConfig(
+            test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json",
+            environments={
+                "__GITLAB__": {"urls": ["http://localhost:8023"]},
+                "__SHOPPING_ADMIN__": {"urls": ["http://localhost:7780/admin"]},
+            },
+        )
+    )
+    return (
+        evaluator.evaluate_task(
+            task_id=task,
+            agent_response={
+                "task_type": task_type,
+                "status": "SUCCESS",
+                "retrieved_data": None,
+            },
+            network_trace=path,
+        ).status
+        == EvalStatus.SUCCESS
+    )
+
+
+@pytest.mark.parametrize("left_target", [False, True])
+def test_navigation_must_end_on_the_requested_page(tmp_path, left_target):
+    entries = [_event("http://localhost:8023/dashboard/todos", navigation=True)]
+    if left_target:
+        entries.append(_event("http://localhost:8023/dashboard/projects", navigation=True))
+    assert _evaluate(tmp_path, 44, entries, "NAVIGATE") is (not left_target)
+
+
+@pytest.mark.parametrize("route", ["issues", "issues/", "issues/123"])
+def test_issue_list_route_does_not_accept_an_issue_detail(tmp_path, route):
+    base = "http://localhost:8023"
+    expected = base + "/a11yproject/a11yproject.com/-/issues/?state=opened&label_name%5B%5D=bug"
+    entries = [
+        _event(
+            base + "/a11yproject/a11yproject.com/-/" + route + "?state=opened&label_name%5B%5D=bug", navigation=True
+        ),
+        _event(base + "/api/graphql", method="POST", referer=expected),
+    ]
+    assert _evaluate(tmp_path, 339, entries, "NAVIGATE") is (route != "issues/123")
+
+
+@pytest.mark.parametrize("readme", [None, "0", "1"])
+def test_empty_project_accepts_unchecked_but_rejects_enabled_readme(tmp_path, readme):
+    form = {"project[name]": "chatgpt_plugin", "project[path]": "chatgpt_plugin", "project[namespace_id]": "2505"}
+    if readme is not None:
+        form["project[initialize_with_readme]"] = readme
+    entries = [_event("http://localhost:8023/projects", method="POST", form=form)]
+    assert _evaluate(tmp_path, 475, entries, "MUTATE") is (readme != "1")
+
+
+@pytest.mark.parametrize(
+    ("task", "name", "action", "amount"),
+    [
+        (699, "spring sale", "by_percent", 20),
+        (700, "fall discount", "cart_fixed", 10),
+    ],
+)
+@pytest.mark.parametrize("customer_group", ["1", "2"])
+def test_price_rule_form_requires_the_requested_customer_group(tmp_path, task, name, action, amount, customer_group):
+    form = {
+        "name": name,
+        "website_ids[0]": "1",
+        "customer_group_ids[0]": customer_group,
+        "simple_action": action,
+        "discount_amount": amount,
+    }
+    entries = [_event("http://localhost:7780/admin/sales_rule/promo_quote/save/", method="POST", form=form)]
+    assert _evaluate(tmp_path, task, entries, "MUTATE") is (customer_group == "1")