From e8b0b114b0f76ee1b1acc5e905a66e8c3cdd8631 Mon Sep 17 00:00:00 2001 From: kevin Date: Mon, 14 Sep 2026 11:35:57 +0800 Subject: [PATCH] Fix verified network event contracts --- assets/dataset/webarena-verified.json | 40 ++++----- assets/dataset/webarna-verfied-hard.json | 7 +- .../evaluators/network_event_evaluator.py | 7 -- tests/api/test_navigation_xhr_evaluator.py | 82 +++++++++++++++++++ .../test_known_contract_corrections.py | 53 ++++++++++++ 5 files changed, 159 insertions(+), 30 deletions(-) create mode 100644 tests/api/test_navigation_xhr_evaluator.py create mode 100644 tests/dataset/test_known_contract_corrections.py diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json index eb73812..50e8234 100644 --- a/assets/dataset/webarena-verified.json +++ b/assets/dataset/webarena-verified.json @@ -8593,7 +8593,7 @@ }, { "evaluator": "NetworkEventEvaluator", - "expected": {"url": "^__GITLAB__/a11yproject/a11yproject.com/-/issues/.*$"}, + "expected": {"url": "^__GITLAB__/a11yproject/a11yproject\\.com/-/issues/?$"}, "ignored_query_params_patterns": [".*"] }, { @@ -8608,7 +8608,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -12232,14 +12232,13 @@ "post_data": { "project[name]": "chatgpt_plugin", "project[namespace_id]": "2505", - "project[path]": "chatgpt_plugin", - "project[initialize_with_readme]": "0" + "project[path]": "chatgpt_plugin" }, "response_status": 302 } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -16695,7 +16694,7 @@ { "evaluator": "NetworkEventEvaluator", "expected": { - "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles"], + "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles", "__REDDIT__/submit/gaming"], "http_method": "POST", "post_data": { "submission[title]": "what is the recommended console to buy these days?", @@ -16705,7 +16704,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -16843,7 +16842,7 @@ { "evaluator": "NetworkEventEvaluator", "expected": { - "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles"], + "url": ["__REDDIT__/submit", "__REDDIT__/submit/consoles", "__REDDIT__/submit/gaming"], "http_method": "POST", "post_data": { "submission[title]": "what is the recommended console to buy these days", @@ -16853,7 +16852,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -16965,7 +16964,7 @@ { "evaluator": "NetworkEventEvaluator", "expected": { - "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning"], + "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning", "__REDDIT__/submit/technology"], "http_method": "POST", "post_data": { "submission[title]": "what is the SOTA web navigation agent repo", @@ -16975,7 +16974,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -17473,7 +17472,7 @@ { "evaluator": "NetworkEventEvaluator", "expected": { - "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning"], + "url": ["__REDDIT__/submit", "__REDDIT__/submit/deeplearning", "__REDDIT__/submit/technology"], "http_method": "POST", "post_data": { "submission[title]": "the effectiveness of deep learning", @@ -17484,7 +17483,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -19058,6 +19057,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19072,7 +19072,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19859,15 +19859,15 @@ "response_status": 302, "post_data": { "name": "spring sale", - "website_ids": [1], - "customer_group_ids": [1], + "website_ids[0]": "1", + "customer_group_ids[0]": "1", "simple_action": "by_percent", "discount_amount": 20 } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19891,15 +19891,15 @@ "response_status": 302, "post_data": { "name": "fall discount", - "website_ids": [1], - "customer_group_ids": [1], + "website_ids[0]": "1", + "customer_group_ids[0]": "1", "simple_action": "cart_fixed", "discount_amount": 10 } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], diff --git a/assets/dataset/webarna-verfied-hard.json b/assets/dataset/webarna-verfied-hard.json index 3aba551..e7ed1e0 100644 --- a/assets/dataset/webarna-verfied-hard.json +++ b/assets/dataset/webarna-verfied-hard.json @@ -10893,7 +10893,8 @@ "expected": { "url": [ "__REDDIT__/submit", - "__REDDIT__/submit/deeplearning" + "__REDDIT__/submit/deeplearning", + "__REDDIT__/submit/technology" ], "headers": null, "query_params": null, @@ -10922,7 +10923,7 @@ "instantiation_dict": { "topic": "the effectiveness of deep learning" }, - "revision": 2 + "revision": 3 }, { "sites": [ @@ -16524,4 +16525,4 @@ }, "revision": 2 } -] \ No newline at end of file +] diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py index 288bbf4..a08ad79 100644 --- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py +++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py @@ -575,13 +575,6 @@ def _filter_events_by_criteria( if not events: return () - # Handle navigate tasks (We only care about the last navigation event) - # For cases where we check navigation events via subsequent XHR/fetch requests, - # we use the normal filtering logic below. - if context.task.is_navigate_task and config.expected.http_method == "GET": - last_navigation_event = [e for e in events if e.is_navigation_event] - return (last_navigation_event[-1],) if last_navigation_event else () - matches = [] try: expected_url: URL = self._normalized_url(config.expected.url, context=context, config=config, strict=True) # type: ignore diff --git a/tests/api/test_navigation_xhr_evaluator.py b/tests/api/test_navigation_xhr_evaluator.py new file mode 100644 index 0000000..4369bc8 --- /dev/null +++ b/tests/api/test_navigation_xhr_evaluator.py @@ -0,0 +1,82 @@ +"""Regression coverage for navigation tasks accepted through an XHR event.""" + +import json +from pathlib import Path +from typing import Any + +from webarena_verified.api import WebArenaVerified +from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.eval import EvalStatus + + +def _entry(url: str, *, referer: str, navigation: bool) -> dict[str, Any]: + headers = [{"name": "Referer", "value": referer}] + if navigation: + headers.extend( + [ + {"name": "Accept", "value": "text/html"}, + {"name": "Sec-Fetch-Dest", "value": "document"}, + {"name": "Sec-Fetch-Mode", "value": "navigate"}, + {"name": "Sec-Fetch-User", "value": "?1"}, + ] + ) + return { + "startedDateTime": "2026-01-01T00:00:00.000Z", + "time": 1, + "request": {"method": "GET", "url": url, "headers": headers, "cookies": [], "queryString": []}, + "response": { + "status": 200, + "headers": [], + "cookies": [], + "content": {"size": 0, "mimeType": "application/json", "text": "{}"}, + "redirectURL": "", + }, + "cache": {}, + "timings": {"send": 0, "wait": 1, "receive": 0}, + } + + +def test_navigate_task_can_be_verified_by_a_matching_xhr(tmp_path: Path) -> None: + base_url = "http://localhost:7780/admin" + trace = { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": [ + _entry( + f"{base_url}/sales/order/", + referer=f"{base_url}/admin/dashboard/", + navigation=True, + ), + _entry( + f"{base_url}/mui/index/render/?namespace=sales_order_grid&search=" + "&keywordUpdated=false&filters%5Bplaceholder%5D=true&filters%5Bstatus%5D=fraud", + referer=f"{base_url}/sales/order/", + navigation=False, + ), + ], + } + } + trace_path = tmp_path / "network.har" + trace_path.write_text(json.dumps(trace)) + + evaluator = WebArenaVerified( + config=WebArenaVerifiedConfig( + environments={ + "__SHOPPING_ADMIN__": { + "urls": [base_url], + "active_url_idx": 0, + "use_header_login": True, + "credentials": {"username": "admin", "password": "admin1234"}, + } + } + ) + ) + result = evaluator.evaluate_task( + task_id=676, + agent_response={"task_type": "NAVIGATE", "status": "SUCCESS", "retrieved_data": None}, + network_trace=trace_path, + ) + + assert result.status == EvalStatus.SUCCESS + assert result.score == 1.0 diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py new file mode 100644 index 0000000..7cfde4f --- /dev/null +++ b/tests/dataset/test_known_contract_corrections.py @@ -0,0 +1,53 @@ +"""Regression tests for evaluator contracts corrected from observed valid traces.""" + +from typing import Any + + +def _network_expectations(task: dict[str, Any]) -> list[dict[str, Any]]: + return [item["expected"] for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + + +def test_issue_list_route_accepts_the_query_bearing_page_url( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + route = _network_expectations(dataset_by_task_id[339])[0]["url"] + assert route == r"^__GITLAB__/a11yproject/a11yproject\.com/-/issues/?$" + + +def test_empty_project_does_not_require_an_unchecked_checkbox_field( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + post_data = _network_expectations(dataset_by_task_id[475])[0]["post_data"] + assert "project[initialize_with_readme]" not in post_data + + +def test_reddit_contracts_accept_the_route_bound_to_the_required_forum_id( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + expected_routes = { + 600: "__REDDIT__/submit/gaming", + 605: "__REDDIT__/submit/gaming", + 609: "__REDDIT__/submit/technology", + 625: "__REDDIT__/submit/technology", + } + for task_id, route in expected_routes.items(): + assert route in _network_expectations(dataset_by_task_id[task_id])[0]["url"] + + +def test_order_grid_xhr_is_not_replaced_by_the_last_same_route_event( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[676] + network_evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert network_evaluator["last_event_only"] is False + + +def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (699, 700): + post_data = _network_expectations(dataset_by_task_id[task_id])[0]["post_data"] + assert post_data["website_ids[0]"] == "1" + assert post_data["customer_group_ids[0]"] == "1" + assert "website_ids" not in post_data + assert "customer_group_ids" not in post_data