ServiceNow · lanyue-llk · Sep 14, 2026
diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json
@@ -8593,7 +8593,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
-        "expected": {"url": "^__GITLAB__/a11yproject/a11yproject.com/-/issues/.*$"},
+        "expected": {"url": "^__GITLAB__/a11yproject/a11yproject\\.com/-/issues/?$"},
         "ignored_query_params_patterns": [".*"]
       },
       {
@@ -8608,7 +8608,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -12232,14 +12232,13 @@
           "post_data": {
             "project[name]": "chatgpt_plugin",
             "project[namespace_id]": "2505",
-            "project[path]": "chatgpt_plugin",
-            "project[initialize_with_readme]": "0"
+            "project[path]": "chatgpt_plugin"
           },
           "response_status": 302
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -16695,7 +16694,7 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "expected": {
-          "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles"],
+          "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles", "__REDDIT__/submit/gaming"],
           "http_method": "POST",
           "post_data": {
             "submission[title]": "what is the recommended console to buy these days?",
@@ -16705,7 +16704,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["reddit"],
@@ -16843,7 +16842,7 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "expected": {
-          "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles"],
+          "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles", "__REDDIT__/submit/gaming"],
           "http_method": "POST",
           "post_data": {
             "submission[title]": "what is the recommended console to buy these days",
@@ -16853,7 +16852,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["reddit"],
@@ -16965,7 +16964,7 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "expected": {
-          "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning"],
+          "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning", "__REDDIT__/submit/technology"],
           "http_method": "POST",
           "post_data": {
             "submission[title]": "what is the SOTA web navigation agent repo",
@@ -16975,7 +16974,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["reddit"],
@@ -17473,7 +17472,7 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "expected": {
-          "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning"],
+          "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning", "__REDDIT__/submit/technology"],
           "http_method": "POST",
           "post_data": {
             "submission[title]": "the effectiveness of deep learning",
@@ -17484,7 +17483,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["reddit"],
@@ -19058,6 +19057,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$",
           "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"},
@@ -19072,7 +19072,7 @@
         "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"]
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19859,15 +19859,15 @@
           "response_status": 302,
           "post_data": {
             "name": "spring sale",
-            "website_ids": [1],
-            "customer_group_ids": [1],
+            "website_ids[0]": "1",
+            "customer_group_ids[0]": "1",
             "simple_action": "by_percent",
             "discount_amount": 20
           }
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19891,15 +19891,15 @@
           "response_status": 302,
           "post_data": {
             "name": "fall discount",
-            "website_ids": [1],
-            "customer_group_ids": [1],
+            "website_ids[0]": "1",
+            "customer_group_ids[0]": "1",
             "simple_action": "cart_fixed",
             "discount_amount": 10
           }
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],

diff --git a/assets/dataset/webarna-verfied-hard.json b/assets/dataset/webarna-verfied-hard.json
@@ -10893,7 +10893,8 @@
         "expected": {
           "url": [
             "__REDDIT__/submit",
-            "__REDDIT__/submit/deeplearning"
+            "__REDDIT__/submit/deeplearning",
+            "__REDDIT__/submit/technology"
           ],
           "headers": null,
           "query_params": null,
@@ -10922,7 +10923,7 @@
     "instantiation_dict": {
       "topic": "the effectiveness of deep learning"
     },
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": [
@@ -16524,4 +16525,4 @@
     },
     "revision": 2
   }
-]
+]
diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
@@ -575,13 +575,6 @@ def _filter_events_by_criteria(
         if not events:
             return ()
 
-        # Handle navigate tasks (We only care about the last navigation event)
-        # For cases where we check navigation events via subsequent XHR/fetch requests,
-        # we use the normal filtering logic below.
-        if context.task.is_navigate_task and config.expected.http_method == "GET":
-            last_navigation_event = [e for e in events if e.is_navigation_event]
-            return (last_navigation_event[-1],) if last_navigation_event else ()
-
         matches = []
         try:
             expected_url: URL = self._normalized_url(config.expected.url, context=context, config=config, strict=True)  # type: ignore

diff --git a/tests/api/test_navigation_xhr_evaluator.py b/tests/api/test_navigation_xhr_evaluator.py
@@ -0,0 +1,82 @@
+"""Regression coverage for navigation tasks accepted through an XHR event."""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from webarena_verified.api import WebArenaVerified
+from webarena_verified.types.config import WebArenaVerifiedConfig
+from webarena_verified.types.eval import EvalStatus
+
+
+def _entry(url: str, *, referer: str, navigation: bool) -> dict[str, Any]:
+    headers = [{"name": "Referer", "value": referer}]
+    if navigation:
+        headers.extend(
+            [
+                {"name": "Accept", "value": "text/html"},
+                {"name": "Sec-Fetch-Dest", "value": "document"},
+                {"name": "Sec-Fetch-Mode", "value": "navigate"},
+                {"name": "Sec-Fetch-User", "value": "?1"},
+            ]
+        )
+    return {
+        "startedDateTime": "2026-01-01T00:00:00.000Z",
+        "time": 1,
+        "request": {"method": "GET", "url": url, "headers": headers, "cookies": [], "queryString": []},
+        "response": {
+            "status": 200,
+            "headers": [],
+            "cookies": [],
+            "content": {"size": 0, "mimeType": "application/json", "text": "{}"},
+            "redirectURL": "",
+        },
+        "cache": {},
+        "timings": {"send": 0, "wait": 1, "receive": 0},
+    }
+
+
+def test_navigate_task_can_be_verified_by_a_matching_xhr(tmp_path: Path) -> None:
+    base_url = "http://localhost:7780/admin"
+    trace = {
+        "log": {
+            "version": "1.2",
+            "creator": {"name": "pytest", "version": "1"},
+            "entries": [
+                _entry(
+                    f"{base_url}/sales/order/",
+                    referer=f"{base_url}/admin/dashboard/",
+                    navigation=True,
+                ),
+                _entry(
+                    f"{base_url}/mui/index/render/?namespace=sales_order_grid&search="
+                    "&keywordUpdated=false&filters%5Bplaceholder%5D=true&filters%5Bstatus%5D=fraud",
+                    referer=f"{base_url}/sales/order/",
+                    navigation=False,
+                ),
+            ],
+        }
+    }
+    trace_path = tmp_path / "network.har"
+    trace_path.write_text(json.dumps(trace))
+
+    evaluator = WebArenaVerified(
+        config=WebArenaVerifiedConfig(
+            environments={
+                "__SHOPPING_ADMIN__": {
+                    "urls": [base_url],
+                    "active_url_idx": 0,
+                    "use_header_login": True,
+                    "credentials": {"username": "admin", "password": "admin1234"},
+                }
+            }
+        )
+    )
+    result = evaluator.evaluate_task(
+        task_id=676,
+        agent_response={"task_type": "NAVIGATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace_path,
+    )
+
+    assert result.status == EvalStatus.SUCCESS
+    assert result.score == 1.0
diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py
@@ -0,0 +1,53 @@
+"""Regression tests for evaluator contracts corrected from observed valid traces."""
+
+from typing import Any
+
+
+def _network_expectations(task: dict[str, Any]) -> list[dict[str, Any]]:
+    return [item["expected"] for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"]
+
+
+def test_issue_list_route_accepts_the_query_bearing_page_url(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    route = _network_expectations(dataset_by_task_id[339])[0]["url"]
+    assert route == r"^__GITLAB__/a11yproject/a11yproject\.com/-/issues/?$"
+
+
+def test_empty_project_does_not_require_an_unchecked_checkbox_field(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    post_data = _network_expectations(dataset_by_task_id[475])[0]["post_data"]
+    assert "project[initialize_with_readme]" not in post_data
+
+
+def test_reddit_contracts_accept_the_route_bound_to_the_required_forum_id(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    expected_routes = {
+        600: "__REDDIT__/submit/gaming",
+        605: "__REDDIT__/submit/gaming",
+        609: "__REDDIT__/submit/technology",
+        625: "__REDDIT__/submit/technology",
+    }
+    for task_id, route in expected_routes.items():
+        assert route in _network_expectations(dataset_by_task_id[task_id])[0]["url"]
+
+
+def test_order_grid_xhr_is_not_replaced_by_the_last_same_route_event(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    task = dataset_by_task_id[676]
+    network_evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
+    assert network_evaluator["last_event_only"] is False
+
+
+def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    for task_id in (699, 700):
+        post_data = _network_expectations(dataset_by_task_id[task_id])[0]["post_data"]
+        assert post_data["website_ids[0]"] == "1"
+        assert post_data["customer_group_ids[0]"] == "1"
+        assert "website_ids" not in post_data
+        assert "customer_group_ids" not in post_data