From 2db426ad6b56b4da1f5315026285c23f9aeeb71a Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:10:03 +0800
Subject: [PATCH 1/6] Fix runtime-bound evaluator contracts

---
 .../core/evaluation/data_types/base.py        |  15 +-
 .../evaluation/data_types/markdown_string.py  |   2 +-
 .../evaluators/network_event_evaluator.py     | 165 +++++++++++++++++-
 .../test_network_event_contracts.py           | 165 ++++++++++++++++++
 4 files changed, 339 insertions(+), 8 deletions(-)
 create mode 100644 tests/core/evaluation/evaluators/test_network_event_contracts.py

diff --git a/src/webarena_verified/core/evaluation/data_types/base.py b/src/webarena_verified/core/evaluation/data_types/base.py
index dc97022..ef34ec6 100644
--- a/src/webarena_verified/core/evaluation/data_types/base.py
+++ b/src/webarena_verified/core/evaluation/data_types/base.py
@@ -244,10 +244,7 @@ def _normalize_string(self, s: str) -> str:
 
         # Step 0: Try to derender URLs (best effort, no exceptions)
         # Let the derender_url_fct decide whether to process the string
-        if self.derender_url_fct is not None and "http" in s:
-            with contextlib.suppress(Exception):
-                s = self.derender_url_fct(s, strict=False)
-                # If derendering fails for any reason, continue with original string
+        s = self._derender_urls(s)
 
         # Step 1: Remove common non-semantic marks BEFORE NFKC normalization
         # U+2122 ™, U+00AE ®, U+2120 ℠, U+00A9 ©
@@ -286,3 +283,13 @@ def _normalize_string(self, s: str) -> str:
 
         # Step 5: Casefold for robust case-insensitive comparison
         return s.casefold()
+
+    def _derender_urls(self, value: str) -> str:
+        """Replace deployment URLs through the configured best-effort callback."""
+        if self.derender_url_fct is None or "http" not in value:
+            return value
+        with contextlib.suppress(Exception):
+            # Evaluators bind the desired strictness when they construct this
+            # callback. Passing it again breaks functools.partial callbacks.
+            return self.derender_url_fct(value)
+        return value
diff --git a/src/webarena_verified/core/evaluation/data_types/markdown_string.py b/src/webarena_verified/core/evaluation/data_types/markdown_string.py
index 34a0054..1ce4f73 100644
--- a/src/webarena_verified/core/evaluation/data_types/markdown_string.py
+++ b/src/webarena_verified/core/evaluation/data_types/markdown_string.py
@@ -40,7 +40,7 @@ def _normalize_string(self, s: str) -> str:
 
         We only strip leading/trailing whitespace.
         """
-        return s.strip()
+        return self._derender_urls(s).strip()
 
     def _type_normalize(self, value: Any) -> str:
         """Normalize markdown string for comparison.
diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
index 32c17c2..a58c379 100644
--- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
+++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
@@ -5,6 +5,7 @@
 """
 
 import re
+from collections.abc import Mapping
 from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, cast
@@ -13,7 +14,7 @@
 from webarena_verified.core.evaluation.data_types import URL
 from webarena_verified.core.utils import logger
 from webarena_verified.core.utils.jsonpath_utils import extract_jsonpath_value, is_jsonpath_key
-from webarena_verified.types.eval import EvalAssertion, EvalStatus, TaskEvalContext
+from webarena_verified.types.eval import EvalAssertion, EvalStatus, EvaluatorResult, TaskEvalContext
 from webarena_verified.types.task import NetworkEventEvaluatorCfg, NetworkEventSpec
 from webarena_verified.types.tracing import NetworkEvent
 
@@ -23,6 +24,67 @@
     from webarena_verified.types.common import SerializableMappingProxyType
 
 
+_PLACEHOLDER = re.compile(r"\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}")
+
+
+def _template_match(template: str, actual: Any, bindings: dict[str, str]) -> bool:
+    """Match a dynamic evaluator template and consistently bind its names."""
+    if not isinstance(actual, (str, int, float)):
+        return False
+    cursor = 0
+    parts = ["^"]
+    local_groups: set[str] = set()
+    for match in _PLACEHOLDER.finditer(template):
+        parts.append(re.escape(template[cursor : match.start()]))
+        name = match.group(1)
+        if name in bindings:
+            parts.append(re.escape(bindings[name]))
+        elif name in local_groups:
+            parts.append(f"(?P={name})")
+        else:
+            parts.append(f"(?P<{name}>[^/?#&]+)")
+            local_groups.add(name)
+        cursor = match.end()
+    parts.extend((re.escape(template[cursor:]), "$"))
+    matched = re.fullmatch("".join(parts), str(actual))
+    if matched is None:
+        return False
+    bindings.update(matched.groupdict())
+    return True
+
+
+def _placeholder_names(value: Any) -> set[str]:
+    if isinstance(value, str):
+        return set(_PLACEHOLDER.findall(value))
+    if isinstance(value, Mapping):
+        names: set[str] = set()
+        for key, item in value.items():
+            names.update(_placeholder_names(key))
+            names.update(_placeholder_names(item))
+        return names
+    if isinstance(value, (list, tuple)):
+        names = set()
+        for item in value:
+            names.update(_placeholder_names(item))
+        return names
+    return set()
+
+
+def _substitute_placeholders(value: Any, bindings: Mapping[str, str]) -> Any:
+    if isinstance(value, str):
+        return _PLACEHOLDER.sub(lambda match: bindings[match.group(1)], value)
+    if isinstance(value, list):
+        return [_substitute_placeholders(item, bindings) for item in value]
+    if isinstance(value, tuple):
+        return tuple(_substitute_placeholders(item, bindings) for item in value)
+    if isinstance(value, Mapping):
+        return {
+            _substitute_placeholders(key, bindings): _substitute_placeholders(item, bindings)
+            for key, item in value.items()
+        }
+    return value
+
+
 class NetworkEventEvaluator(BaseEvaluator[NetworkEventEvaluatorCfg]):
     """Validates network events using four-step architecture.
 
@@ -66,7 +128,8 @@ def _get_actual_value(self, context: TaskEvalContext, config: NetworkEventEvalua
             return []
 
         # Step 2: Select event(s)
-        selected_events = [matching_events[-1]] if config.last_event_only else matching_events
+        use_last_event_only = config.last_event_only and not self._has_repeated_event_contract(context, config)
+        selected_events = [matching_events[-1]] if use_last_event_only else matching_events
 
         # Step 3: Extract relevant data from events
         extracted_data = []
@@ -74,7 +137,19 @@ def _get_actual_value(self, context: TaskEvalContext, config: NetworkEventEvalua
             event_data = self._extract_event_data(event, config, context)
             extracted_data.append(event_data)
 
-        return [extracted_data[0]] if config.last_event_only else extracted_data
+        return [extracted_data[0]] if use_last_event_only else extracted_data
+
+    @staticmethod
+    def _has_repeated_event_contract(context: TaskEvalContext, config: NetworkEventEvaluatorCfg) -> bool:
+        """Return whether sibling assertions target the same request stream."""
+        siblings = [
+            item
+            for item in context.task.eval
+            if isinstance(item, NetworkEventEvaluatorCfg)
+            and item.expected.url == config.expected.url
+            and item.expected.http_method == config.expected.http_method
+        ]
+        return len(siblings) > 1
 
     def _get_expected_value(self, config: NetworkEventEvaluatorCfg) -> NetworkEventSpec:
         """Get expected value from config.
@@ -695,3 +770,87 @@ def _should_ignore_post_param(self, param_name: str, config: NetworkEventEvaluat
                     continue
 
         return False
+
+    def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg) -> EvaluatorResult:
+        """Evaluate each runtime-bound form of a dynamic network contract."""
+        candidates = self._runtime_configs(context, config)
+        if not candidates:
+            return super().evaluate(context=context, config=config)
+
+        first_result: EvaluatorResult | None = None
+        for candidate in candidates:
+            result = super().evaluate(context=context, config=candidate)
+            first_result = first_result or result
+            if result.score == 1:
+                return result
+        assert first_result is not None
+        return first_result
+
+    def _runtime_configs(
+        self, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
+    ) -> list[NetworkEventEvaluatorCfg]:
+        """Bind placeholders from one event, then retain normal evaluation."""
+        expected = config.expected.model_dump(mode="python", exclude_none=False)
+        names = _placeholder_names(expected)
+        candidates: list[NetworkEventEvaluatorCfg] = []
+
+        if names:
+            raw_urls = expected.get("url")
+            url_templates = list(raw_urls) if isinstance(raw_urls, (list, tuple)) else [raw_urls]
+            post_templates = expected.get("post_data") or {}
+            for event in context.network_trace.evaluation_events:
+                expected_method = expected.get("http_method")
+                if expected_method and event.http_method.lower() != expected_method.lower():
+                    continue
+
+                bindings: dict[str, str] = {}
+                dynamic_urls = [item for item in url_templates if isinstance(item, str) and _PLACEHOLDER.search(item)]
+                if dynamic_urls:
+                    actual_url = context.config.derender_url(event.url, sites=context.task.sites, strict=False)
+                    if not any(_template_match(item, actual_url, bindings) for item in dynamic_urls):
+                        continue
+
+                actual_post = dict(event.post_data or {})
+                valid = True
+                for key_template, value_template in post_templates.items():
+                    actual_value = actual_post.get(key_template)
+                    if isinstance(key_template, str) and _PLACEHOLDER.search(key_template):
+                        matched_key = next(
+                            (key for key in actual_post if _template_match(key_template, key, bindings.copy())),
+                            None,
+                        )
+                        if matched_key is None or not _template_match(key_template, matched_key, bindings):
+                            valid = False
+                            break
+                        actual_value = actual_post[matched_key]
+                    if (
+                        isinstance(value_template, str)
+                        and _PLACEHOLDER.search(value_template)
+                        and not _template_match(value_template, actual_value, bindings)
+                    ):
+                        valid = False
+                        break
+                if valid and set(bindings) == names:
+                    corrected_expected = _substitute_placeholders(expected, bindings)
+                    candidates.append(
+                        config.model_copy(update={"expected": config.expected.model_copy(update=corrected_expected)})
+                    )
+        else:
+            candidates = [config]
+
+        return [self._with_inferred_post_schema(candidate) for candidate in candidates]
+
+    @staticmethod
+    def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEventEvaluatorCfg:
+        """Preserve singleton array semantics when the contract omits a schema."""
+        post_data = config.expected.post_data or {}
+        singleton_arrays = {
+            key for key, value in post_data.items() if isinstance(value, (list, tuple)) and len(value) == 1
+        }
+        if not singleton_arrays or config.post_data_schema:
+            return config
+        schema = {
+            "type": "object",
+            "properties": {key: {"type": "array", "items": {"type": "string"}} for key in singleton_arrays},
+        }
+        return config.model_copy(update={"post_data_schema": schema})
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
new file mode 100644
index 0000000..51c3dbb
--- /dev/null
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -0,0 +1,165 @@
+"""Regression coverage for network contracts that depend on runtime events."""
+
+import json
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlencode
+
+import pytest
+
+from webarena_verified import WebArenaVerified
+
+
+def _entry(
+    *,
+    method: str,
+    url: str,
+    post_data: dict[str, Any],
+    status: int,
+    mime_type: str = "application/json",
+) -> dict[str, Any]:
+    body = json.dumps(post_data) if mime_type == "application/json" else urlencode(post_data, doseq=True)
+    return {
+        "startedDateTime": "2026-01-01T00:00:00.000Z",
+        "time": 1,
+        "request": {
+            "method": method,
+            "url": url,
+            "headers": [],
+            "cookies": [],
+            "queryString": [],
+            "postData": {"mimeType": mime_type, "text": body},
+        },
+        "response": {
+            "status": status,
+            "headers": [],
+            "cookies": [],
+            "content": {"size": 0, "mimeType": "application/json", "text": "{}"},
+            "redirectURL": "",
+        },
+        "cache": {},
+        "timings": {"send": 0, "wait": 1, "receive": 0},
+    }
+
+
+def _evaluate(wa: WebArenaVerified, tmp_path: Path, task_id: int, entries: list[dict[str, Any]]):
+    trace = tmp_path / f"task-{task_id}.har"
+    trace.write_text(
+        json.dumps(
+            {
+                "log": {
+                    "version": "1.2",
+                    "creator": {"name": "pytest", "version": "1"},
+                    "entries": entries,
+                }
+            }
+        )
+    )
+    return wa.evaluate_task(
+        task_id=task_id,
+        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        network_trace=trace,
+    )
+
+
+def test_repeated_endpoint_contracts_match_distinct_request_bodies(wa: WebArenaVerified, tmp_path: Path) -> None:
+    task = wa.get_task(567)
+    expected = [item.expected for item in task.network_event_evaluator_cfgs]
+    assert all(isinstance(item.url, str) for item in expected)
+    entries = [
+        _entry(
+            method=item.http_method,
+            url=str(item.url).replace("__GITLAB__", "http://localhost:8023"),
+            post_data=dict(item.post_data or {}),
+            status=item.response_status,
+        )
+        for item in expected
+    ]
+
+    assert float(_evaluate(wa, tmp_path, 567, entries).score) == 1.0
+    assert float(_evaluate(wa, tmp_path, 567, entries[1:]).score) == 0.0
+
+
+@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:9999", 1.0), ("http://wrong.test", 0.0)])
+def test_markdown_form_values_derender_embedded_urls(
+    wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float
+) -> None:
+    task = wa.get_task(562)
+    project, commit = [item.expected for item in task.network_event_evaluator_cfgs]
+    assert project.post_data is not None
+    assert commit.post_data is not None
+    content = commit.post_data["$.actions[0].content"].replace("__REDDIT__", origin)
+    entries = [
+        _entry(
+            method="POST",
+            url="http://localhost:8023/projects",
+            post_data={key: value for key, value in project.post_data.items() if value is not None},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
+        _entry(
+            method="POST",
+            url="http://localhost:8023/api/v4/projects/byteblaze%2FAwesome_DIY_ideas/repository/commits",
+            post_data={
+                "branch": "main",
+                "actions": [{"action": "create", "file_path": "README.md", "content": content}],
+            },
+            status=201,
+        ),
+    ]
+    assert float(_evaluate(wa, tmp_path, 562, entries).score) == expected_score
+
+
+@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:8023", 1.0), ("http://wrong.test", 0.0)])
+def test_plain_form_values_derender_embedded_urls(
+    wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float
+) -> None:
+    expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected
+    assert expected.post_data is not None
+    post_data = dict(expected.post_data)
+    post_data["submission[url]"] = post_data["submission[url]"].replace("__GITLAB__", origin)
+    entries = [
+        _entry(
+            method="POST",
+            url="http://localhost:9999/submit/LifeProTips",
+            post_data=post_data,
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        )
+    ]
+    assert float(_evaluate(wa, tmp_path, 684, entries).score) == expected_score
+
+
+def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVerified, tmp_path: Path) -> None:
+    entries = [
+        _entry(
+            method="POST",
+            url="http://localhost:9999/submit/books",
+            post_data={"submission[title]": "Harry Potter", "submission[forum]": "10037"},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
+        _entry(
+            method="POST",
+            url="http://localhost:9999/f/books/4242/-/comment",
+            post_data={"reply_to_submission_4242[comment]": "Wonderful journey"},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
+    ]
+    assert float(_evaluate(wa, tmp_path, 611, entries).score) == 1.0
+
+    entries[1]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"})
+    assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0
+
+
+def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebArenaVerified, tmp_path: Path) -> None:
+    entries = [
+        _entry(
+            method="PUT",
+            url="http://localhost:8023/a11yproject/a11yproject.com/-/issues/1478.json",
+            post_data={"issue": {"assignee_ids": [2330]}},
+            status=200,
+        )
+    ]
+    assert float(_evaluate(wa, tmp_path, 811, entries).score) == 1.0

From 31d7dae0924dcc75c56b72de785d6c53659bad37 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:19:22 +0800
Subject: [PATCH 2/6] Preserve explicit network event semantics

---
 assets/dataset/webarena-verified.json         |  18 ++-
 .../evaluators/network_event_evaluator.py     | 120 ++++++++++++++----
 .../test_network_event_contracts.py           |  34 ++++-
 .../test_known_contract_corrections.py        |  10 ++
 4 files changed, 153 insertions(+), 29 deletions(-)

diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json
index fe1af37..f71c042 100644
--- a/assets/dataset/webarena-verified.json
+++ b/assets/dataset/webarena-verified.json
@@ -15546,6 +15546,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/184/members",
           "http_method": "POST",
@@ -15555,6 +15556,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/184/members",
           "http_method": "POST",
@@ -15563,7 +15565,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -15581,6 +15583,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/174/members",
           "http_method": "POST",
@@ -15590,6 +15593,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/174/members",
           "http_method": "POST",
@@ -15598,7 +15602,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -15619,6 +15623,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/185/members",
           "http_method": "POST",
@@ -15628,6 +15633,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/185/members",
           "http_method": "POST",
@@ -15636,7 +15642,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -15657,6 +15663,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/190/members",
           "http_method": "POST",
@@ -15666,6 +15673,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/190/members",
           "http_method": "POST",
@@ -15675,6 +15683,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/190/members",
           "http_method": "POST",
@@ -15684,6 +15693,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "__GITLAB__/api/v4/projects/190/members",
           "http_method": "POST",
@@ -15692,7 +15702,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping"],
diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
index a58c379..e2af437 100644
--- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
+++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
@@ -128,8 +128,7 @@ def _get_actual_value(self, context: TaskEvalContext, config: NetworkEventEvalua
             return []
 
         # Step 2: Select event(s)
-        use_last_event_only = config.last_event_only and not self._has_repeated_event_contract(context, config)
-        selected_events = [matching_events[-1]] if use_last_event_only else matching_events
+        selected_events = [matching_events[-1]] if config.last_event_only else matching_events
 
         # Step 3: Extract relevant data from events
         extracted_data = []
@@ -137,19 +136,7 @@ def _get_actual_value(self, context: TaskEvalContext, config: NetworkEventEvalua
             event_data = self._extract_event_data(event, config, context)
             extracted_data.append(event_data)
 
-        return [extracted_data[0]] if use_last_event_only else extracted_data
-
-    @staticmethod
-    def _has_repeated_event_contract(context: TaskEvalContext, config: NetworkEventEvaluatorCfg) -> bool:
-        """Return whether sibling assertions target the same request stream."""
-        siblings = [
-            item
-            for item in context.task.eval
-            if isinstance(item, NetworkEventEvaluatorCfg)
-            and item.expected.url == config.expected.url
-            and item.expected.http_method == config.expected.http_method
-        ]
-        return len(siblings) > 1
+        return [extracted_data[0]] if config.last_event_only else extracted_data
 
     def _get_expected_value(self, config: NetworkEventEvaluatorCfg) -> NetworkEventSpec:
         """Get expected value from config.
@@ -331,6 +318,16 @@ def _normalize_post_data_field(
         if post_data_dict:
             post_data_schema = config.post_data_schema or {}
 
+            if (
+                not strict
+                and post_data_schema.get("x-inferred-singleton-arrays")
+                and not self._matches_json_schema(post_data_dict, post_data_schema)
+            ):
+                # Preserve the mismatched raw value so structural comparison
+                # rejects it instead of coercing (for example, "2330" to 2330).
+                normalized["post_data"] = post_data_dict
+                return
+
             _derender_url_fct = partial(context.config.derender_url, sites=context.task.sites, strict=strict)
             normalized["post_data"] = self.value_normalizer.normalize(
                 post_data_dict, post_data_schema, strict=strict, derender_url_fct=_derender_url_fct
@@ -778,8 +775,13 @@ def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
             return super().evaluate(context=context, config=config)
 
         first_result: EvaluatorResult | None = None
-        for candidate in candidates:
-            result = super().evaluate(context=context, config=candidate)
+        for candidate, source_event in candidates:
+            candidate_context = context
+            if source_event is not None:
+                candidate_context = context.model_copy(
+                    update={"network_trace": context.network_trace.model_copy(update={"events": (source_event,)})}
+                )
+            result = super().evaluate(context=candidate_context, config=candidate)
             first_result = first_result or result
             if result.score == 1:
                 return result
@@ -788,11 +790,11 @@ def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
 
     def _runtime_configs(
         self, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
-    ) -> list[NetworkEventEvaluatorCfg]:
-        """Bind placeholders from one event, then retain normal evaluation."""
+    ) -> list[tuple[NetworkEventEvaluatorCfg, NetworkEvent | None]]:
+        """Bind placeholders and retain the event that supplied each binding."""
         expected = config.expected.model_dump(mode="python", exclude_none=False)
         names = _placeholder_names(expected)
-        candidates: list[NetworkEventEvaluatorCfg] = []
+        candidates: list[tuple[NetworkEventEvaluatorCfg, NetworkEvent | None]] = []
 
         if names:
             raw_urls = expected.get("url")
@@ -833,12 +835,17 @@ def _runtime_configs(
                 if valid and set(bindings) == names:
                     corrected_expected = _substitute_placeholders(expected, bindings)
                     candidates.append(
-                        config.model_copy(update={"expected": config.expected.model_copy(update=corrected_expected)})
+                        (
+                            config.model_copy(
+                                update={"expected": config.expected.model_copy(update=corrected_expected)}
+                            ),
+                            event,
+                        )
                     )
         else:
-            candidates = [config]
+            candidates = [(config, None)]
 
-        return [self._with_inferred_post_schema(candidate) for candidate in candidates]
+        return [(self._with_inferred_post_schema(candidate), source_event) for candidate, source_event in candidates]
 
     @staticmethod
     def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEventEvaluatorCfg:
@@ -851,6 +858,71 @@ def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEvent
             return config
         schema = {
             "type": "object",
-            "properties": {key: {"type": "array", "items": {"type": "string"}} for key in singleton_arrays},
+            "properties": {
+                key: {
+                    "type": "array",
+                    "items": NetworkEventEvaluator._schema_for_json_value(post_data[key][0]),
+                }
+                for key in singleton_arrays
+            },
+            "x-inferred-singleton-arrays": True,
         }
         return config.model_copy(update={"post_data_schema": schema})
+
+    @staticmethod
+    def _schema_for_json_value(value: Any) -> dict[str, Any]:
+        """Infer a JSON schema without coercing distinct scalar types."""
+        if value is None:
+            schema = {"type": "null"}
+        elif isinstance(value, bool):
+            schema = {"type": "boolean"}
+        elif isinstance(value, int):
+            schema = {"type": "integer"}
+        elif isinstance(value, float):
+            schema = {"type": "number"}
+        elif isinstance(value, str):
+            schema = {"type": "string"}
+        elif isinstance(value, Mapping):
+            schema = {
+                "type": "object",
+                "properties": {key: NetworkEventEvaluator._schema_for_json_value(item) for key, item in value.items()},
+            }
+        elif isinstance(value, (list, tuple)):
+            schema = {
+                "type": "array",
+                "items": NetworkEventEvaluator._schema_for_json_value(value[0]) if value else {},
+            }
+        else:
+            raise TypeError(f"Cannot infer a POST data schema for {type(value).__name__}")
+        return schema
+
+    @staticmethod
+    def _matches_json_schema(value: Any, schema: Mapping[str, Any]) -> bool:
+        """Check the JSON types used by inferred singleton-array schemas."""
+        schema_type = schema.get("type")
+        if schema_type == "null":
+            matches = value is None
+        elif schema_type == "boolean":
+            matches = isinstance(value, bool)
+        elif schema_type == "integer":
+            matches = isinstance(value, int) and not isinstance(value, bool)
+        elif schema_type == "number":
+            matches = isinstance(value, (int, float)) and not isinstance(value, bool)
+        elif schema_type == "string":
+            matches = isinstance(value, str)
+        elif schema_type == "array":
+            item_schema = schema.get("items", {})
+            matches = isinstance(value, (list, tuple)) and all(
+                NetworkEventEvaluator._matches_json_schema(item, item_schema) for item in value
+            )
+        elif schema_type == "object":
+            if isinstance(value, Mapping):
+                matches = all(
+                    key in value and NetworkEventEvaluator._matches_json_schema(value[key], property_schema)
+                    for key, property_schema in schema.get("properties", {}).items()
+                )
+            else:
+                matches = False
+        else:
+            matches = True
+        return matches
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
index 51c3dbb..97d241d 100644
--- a/tests/core/evaluation/evaluators/test_network_event_contracts.py
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -130,6 +130,28 @@ def test_plain_form_values_derender_embedded_urls(
     assert float(_evaluate(wa, tmp_path, 684, entries).score) == expected_score
 
 
+def test_last_event_only_rejects_a_later_wrong_request_to_the_same_endpoint(
+    wa: WebArenaVerified, tmp_path: Path
+) -> None:
+    expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected
+    assert expected.post_data is not None
+    correct = dict(expected.post_data)
+    correct["submission[url]"] = correct["submission[url]"].replace("__GITLAB__", "http://localhost:8023")
+    wrong = {**correct, "submission[title]": "wrong title"}
+    entries = [
+        _entry(
+            method="POST",
+            url="http://localhost:9999/submit/LifeProTips",
+            post_data=post_data,
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        )
+        for post_data in (correct, wrong)
+    ]
+
+    assert float(_evaluate(wa, tmp_path, 684, entries).score) == 0.0
+
+
 def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVerified, tmp_path: Path) -> None:
     entries = [
         _entry(
@@ -139,6 +161,13 @@ def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVe
             status=302,
             mime_type="application/x-www-form-urlencoded",
         ),
+        _entry(
+            method="POST",
+            url="http://localhost:9999/f/books/1111/-/comment",
+            post_data={"reply_to_submission_2222[comment]": "Wonderful journey"},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
         _entry(
             method="POST",
             url="http://localhost:9999/f/books/4242/-/comment",
@@ -149,7 +178,7 @@ def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVe
     ]
     assert float(_evaluate(wa, tmp_path, 611, entries).score) == 1.0
 
-    entries[1]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"})
+    entries[2]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"})
     assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0
 
 
@@ -163,3 +192,6 @@ def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebAre
         )
     ]
     assert float(_evaluate(wa, tmp_path, 811, entries).score) == 1.0
+
+    entries[0]["request"]["postData"]["text"] = json.dumps({"issue": {"assignee_ids": ["2330"]}})
+    assert float(_evaluate(wa, tmp_path, 811, entries).score) == 0.0
diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py
index 12c2f64..82812b1 100644
--- a/tests/dataset/test_known_contract_corrections.py
+++ b/tests/dataset/test_known_contract_corrections.py
@@ -39,3 +39,13 @@ def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives(
         assert post_data["customer_group_ids[0]"] == "1"
         assert "website_ids" not in post_data
         assert "customer_group_ids" not in post_data
+
+
+def test_repeated_member_requests_explicitly_match_any_event(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    for task_id in (567, 568, 569, 570):
+        task = dataset_by_task_id[task_id]
+        evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"]
+        assert task["revision"] == 3
+        assert all(item["last_event_only"] is False for item in evaluators)

From 0f2af09de732697baeb94d151251a8bf77032fd0 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:25:33 +0800
Subject: [PATCH 3/6] Honor resolved event stream ordering

---
 assets/dataset/webarena-verified.json         | 26 +++++++++---
 .../evaluators/network_event_evaluator.py     | 41 ++++++++++++------
 .../test_network_event_contracts.py           | 42 +++++++++++++++++++
 .../test_known_contract_corrections.py        | 23 ++++++++++
 4 files changed, 115 insertions(+), 17 deletions(-)

diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json
index f71c042..3add3f0 100644
--- a/assets/dataset/webarena-verified.json
+++ b/assets/dataset/webarena-verified.json
@@ -21523,6 +21523,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21532,6 +21533,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21540,7 +21542,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -21571,6 +21573,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21580,6 +21583,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21588,7 +21592,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -21658,6 +21662,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21667,6 +21672,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21676,6 +21682,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21684,7 +21691,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -21715,6 +21722,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21724,6 +21732,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21733,6 +21742,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": false,
         "expected": {
           "url": "^__GITLAB__/api/v4/projects/\\d+/members$",
           "http_method": "POST",
@@ -21741,7 +21751,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -23961,6 +23971,12 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "ignored_query_params": ["serializer"],
+        "post_data_schema": {
+          "type": "object",
+          "properties": {
+            "$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}
+          }
+        },
         "expected": {
           "url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1478.json",
           "http_method": "PUT",
@@ -23968,6 +23984,6 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   }
 ]
diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
index e2af437..2177ab1 100644
--- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
+++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
@@ -320,7 +320,7 @@ def _normalize_post_data_field(
 
             if (
                 not strict
-                and post_data_schema.get("x-inferred-singleton-arrays")
+                and self._contains_array_schema(post_data_schema)
                 and not self._matches_json_schema(post_data_dict, post_data_schema)
             ):
                 # Preserve the mismatched raw value so structural comparison
@@ -775,11 +775,11 @@ def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
             return super().evaluate(context=context, config=config)
 
         first_result: EvaluatorResult | None = None
-        for candidate, source_event in candidates:
+        for candidate, source_events in candidates:
             candidate_context = context
-            if source_event is not None:
+            if source_events is not None:
                 candidate_context = context.model_copy(
-                    update={"network_trace": context.network_trace.model_copy(update={"events": (source_event,)})}
+                    update={"network_trace": context.network_trace.model_copy(update={"events": source_events})}
                 )
             result = super().evaluate(context=candidate_context, config=candidate)
             first_result = first_result or result
@@ -790,11 +790,13 @@ def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
 
     def _runtime_configs(
         self, context: TaskEvalContext, config: NetworkEventEvaluatorCfg
-    ) -> list[tuple[NetworkEventEvaluatorCfg, NetworkEvent | None]]:
-        """Bind placeholders and retain the event that supplied each binding."""
+    ) -> list[tuple[NetworkEventEvaluatorCfg, tuple[NetworkEvent, ...] | None]]:
+        """Bind placeholders and group ordered events by resolved binding."""
         expected = config.expected.model_dump(mode="python", exclude_none=False)
         names = _placeholder_names(expected)
-        candidates: list[tuple[NetworkEventEvaluatorCfg, NetworkEvent | None]] = []
+        candidates_by_binding: dict[
+            tuple[tuple[str, str], ...], tuple[NetworkEventEvaluatorCfg, list[NetworkEvent]]
+        ] = {}
 
         if names:
             raw_urls = expected.get("url")
@@ -834,14 +836,19 @@ def _runtime_configs(
                         break
                 if valid and set(bindings) == names:
                     corrected_expected = _substitute_placeholders(expected, bindings)
-                    candidates.append(
-                        (
+                    binding_key = tuple(sorted(bindings.items()))
+                    if binding_key not in candidates_by_binding:
+                        candidates_by_binding[binding_key] = (
                             config.model_copy(
                                 update={"expected": config.expected.model_copy(update=corrected_expected)}
                             ),
-                            event,
+                            [],
                         )
-                    )
+                    candidates_by_binding[binding_key][1].append(event)
+
+            candidates = [
+                (candidate, tuple(source_events)) for candidate, source_events in candidates_by_binding.values()
+            ]
         else:
             candidates = [(config, None)]
 
@@ -865,7 +872,6 @@ def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEvent
                 }
                 for key in singleton_arrays
             },
-            "x-inferred-singleton-arrays": True,
         }
         return config.model_copy(update={"post_data_schema": schema})
 
@@ -926,3 +932,14 @@ def _matches_json_schema(value: Any, schema: Mapping[str, Any]) -> bool:
         else:
             matches = True
         return matches
+
+    @staticmethod
+    def _contains_array_schema(schema: Mapping[str, Any]) -> bool:
+        """Return whether a schema contains an array at any depth."""
+        if schema.get("type") == "array":
+            return True
+        properties = schema.get("properties", {})
+        return isinstance(properties, Mapping) and any(
+            isinstance(property_schema, Mapping) and NetworkEventEvaluator._contains_array_schema(property_schema)
+            for property_schema in properties.values()
+        )
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
index 97d241d..d7a136f 100644
--- a/tests/core/evaluation/evaluators/test_network_event_contracts.py
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -80,6 +80,31 @@ def test_repeated_endpoint_contracts_match_distinct_request_bodies(wa: WebArenaV
     assert float(_evaluate(wa, tmp_path, 567, entries[1:]).score) == 0.0
 
 
+@pytest.mark.parametrize("task_id", [742, 743, 745, 746])
+def test_project_creation_with_multiple_members_matches_complete_trace(
+    wa: WebArenaVerified, tmp_path: Path, task_id: int
+) -> None:
+    task = wa.get_task(task_id)
+    entries = []
+    for config in task.network_event_evaluator_cfgs:
+        expected = config.expected
+        url = (
+            "http://localhost:8023/api/v4/projects"
+            if expected.url == "__GITLAB__/api/v4/projects"
+            else "http://localhost:8023/api/v4/projects/4242/members"
+        )
+        entries.append(
+            _entry(
+                method=expected.http_method,
+                url=url,
+                post_data=dict(expected.post_data or {}),
+                status=expected.response_status,
+            )
+        )
+
+    assert float(_evaluate(wa, tmp_path, task_id, entries).score) == 1.0
+
+
 @pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:9999", 1.0), ("http://wrong.test", 0.0)])
 def test_markdown_form_values_derender_embedded_urls(
     wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float
@@ -178,11 +203,28 @@ def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVe
     ]
     assert float(_evaluate(wa, tmp_path, 611, entries).score) == 1.0
 
+    entries.append(
+        _entry(
+            method="POST",
+            url="http://localhost:9999/f/books/4242/-/comment",
+            post_data={"reply_to_submission_4242[comment]": "wrong final comment"},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        )
+    )
+    assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0
+
+    entries.pop()
     entries[2]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"})
     assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0
 
 
 def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebArenaVerified, tmp_path: Path) -> None:
+    config = wa.get_task(811).network_event_evaluator_cfgs[0]
+    assert config.post_data_schema == {
+        "type": "object",
+        "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}},
+    }
     entries = [
         _entry(
             method="PUT",
diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py
index 82812b1..f50da5a 100644
--- a/tests/dataset/test_known_contract_corrections.py
+++ b/tests/dataset/test_known_contract_corrections.py
@@ -49,3 +49,26 @@ def test_repeated_member_requests_explicitly_match_any_event(
         evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"]
         assert task["revision"] == 3
         assert all(item["last_event_only"] is False for item in evaluators)
+
+    for task_id in (742, 743, 745, 746):
+        task = dataset_by_task_id[task_id]
+        member_evaluators = [
+            item
+            for item in task["eval"]
+            if item["evaluator"] == "NetworkEventEvaluator" and item["expected"]["url"].endswith("/members$")
+        ]
+        assert task["revision"] == 3
+        assert len(member_evaluators) > 1
+        assert all(item["last_event_only"] is False for item in member_evaluators)
+
+
+def test_issue_assignee_array_has_an_explicit_integer_contract(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    task = dataset_by_task_id[811]
+    evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
+    assert task["revision"] == 3
+    assert evaluator["post_data_schema"] == {
+        "type": "object",
+        "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}},
+    }

From 803de6a7313452f65fd026070127a34a02b92022 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:32:38 +0800
Subject: [PATCH 4/6] Keep dynamic POST bindings in one event stream

---
 .../evaluators/network_event_evaluator.py     | 106 ++++++++++++------
 .../test_network_event_contracts.py           |  24 ++++
 2 files changed, 98 insertions(+), 32 deletions(-)

diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
index 2177ab1..4ad252a 100644
--- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
+++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py
@@ -795,8 +795,9 @@ def _runtime_configs(
         expected = config.expected.model_dump(mode="python", exclude_none=False)
         names = _placeholder_names(expected)
         candidates_by_binding: dict[
-            tuple[tuple[str, str], ...], tuple[NetworkEventEvaluatorCfg, list[NetworkEvent]]
+            tuple[tuple[str, str], ...], tuple[NetworkEventEvaluatorCfg, tuple[tuple[str, str], ...]]
         ] = {}
+        events_by_stream: dict[tuple[tuple[str, str], ...], list[NetworkEvent]] = {}
 
         if names:
             raw_urls = expected.get("url")
@@ -807,34 +808,20 @@ def _runtime_configs(
                 if expected_method and event.http_method.lower() != expected_method.lower():
                     continue
 
-                bindings: dict[str, str] = {}
-                dynamic_urls = [item for item in url_templates if isinstance(item, str) and _PLACEHOLDER.search(item)]
-                if dynamic_urls:
-                    actual_url = context.config.derender_url(event.url, sites=context.task.sites, strict=False)
-                    if not any(_template_match(item, actual_url, bindings) for item in dynamic_urls):
-                        continue
-
-                actual_post = dict(event.post_data or {})
-                valid = True
-                for key_template, value_template in post_templates.items():
-                    actual_value = actual_post.get(key_template)
-                    if isinstance(key_template, str) and _PLACEHOLDER.search(key_template):
-                        matched_key = next(
-                            (key for key in actual_post if _template_match(key_template, key, bindings.copy())),
-                            None,
-                        )
-                        if matched_key is None or not _template_match(key_template, matched_key, bindings):
-                            valid = False
-                            break
-                        actual_value = actual_post[matched_key]
-                    if (
-                        isinstance(value_template, str)
-                        and _PLACEHOLDER.search(value_template)
-                        and not _template_match(value_template, actual_value, bindings)
-                    ):
-                        valid = False
-                        break
-                if valid and set(bindings) == names:
+                stream = self._resolve_event_stream(
+                    event=event,
+                    context=context,
+                    config=config,
+                    raw_urls=raw_urls,
+                    url_templates=url_templates,
+                )
+                if stream is None:
+                    continue
+                stream_key, bindings = stream
+                events_by_stream.setdefault(stream_key, []).append(event)
+
+                bindings = self._bind_post_placeholders(post_templates, event.post_data or {}, bindings)
+                if bindings is not None and set(bindings) == names:
                     corrected_expected = _substitute_placeholders(expected, bindings)
                     binding_key = tuple(sorted(bindings.items()))
                     if binding_key not in candidates_by_binding:
@@ -842,18 +829,73 @@ def _runtime_configs(
                             config.model_copy(
                                 update={"expected": config.expected.model_copy(update=corrected_expected)}
                             ),
-                            [],
+                            stream_key,
                         )
-                    candidates_by_binding[binding_key][1].append(event)
 
             candidates = [
-                (candidate, tuple(source_events)) for candidate, source_events in candidates_by_binding.values()
+                (candidate, tuple(events_by_stream[stream_key]))
+                for candidate, stream_key in candidates_by_binding.values()
             ]
         else:
             candidates = [(config, None)]
 
         return [(self._with_inferred_post_schema(candidate), source_event) for candidate, source_event in candidates]
 
+    def _resolve_event_stream(
+        self,
+        *,
+        event: NetworkEvent,
+        context: TaskEvalContext,
+        config: NetworkEventEvaluatorCfg,
+        raw_urls: Any,
+        url_templates: list[Any],
+    ) -> tuple[tuple[tuple[str, str], ...], dict[str, str]] | None:
+        """Resolve URL bindings and apply the evaluator's normal prefilter."""
+        bindings: dict[str, str] = {}
+        dynamic_urls = [item for item in url_templates if isinstance(item, str) and _PLACEHOLDER.search(item)]
+        resolved_urls = raw_urls
+        if dynamic_urls:
+            actual_url = context.config.derender_url(event.url, sites=context.task.sites, strict=False)
+            for item in dynamic_urls:
+                candidate_bindings = bindings.copy()
+                if _template_match(item, actual_url, candidate_bindings):
+                    bindings = candidate_bindings
+                    resolved_urls = _substitute_placeholders(item, bindings)
+                    break
+            else:
+                return None
+
+        filter_config = config.model_copy(
+            update={"expected": config.expected.model_copy(update={"url": resolved_urls})}
+        )
+        if not self._filter_events_by_criteria((event,), context, filter_config):
+            return None
+        return tuple(sorted(bindings.items())), bindings
+
+    @staticmethod
+    def _bind_post_placeholders(
+        post_templates: Mapping[str, Any], actual_post: Mapping[str, Any], initial: Mapping[str, str]
+    ) -> dict[str, str] | None:
+        """Bind dynamic POST keys and values without defining an event stream."""
+        bindings = dict(initial)
+        for key_template, value_template in post_templates.items():
+            actual_value = actual_post.get(key_template)
+            if _PLACEHOLDER.search(key_template):
+                matched_key = next(
+                    (key for key in actual_post if _template_match(key_template, key, bindings.copy())),
+                    None,
+                )
+                if matched_key is None or not _template_match(key_template, matched_key, bindings):
+                    return None
+                actual_value = actual_post[matched_key]
+            if (
+                isinstance(value_template, str)
+                and _PLACEHOLDER.search(value_template)
+                and not _template_match(value_template, actual_value, bindings)
+            ):
+                return None
+        return bindings
+
     @staticmethod
     def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEventEvaluatorCfg:
         """Preserve singleton array semantics when the contract omits a schema."""
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
index d7a136f..cd6934c 100644
--- a/tests/core/evaluation/evaluators/test_network_event_contracts.py
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -219,6 +219,30 @@ def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVe
     assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0
 
 
+def test_dynamic_post_binding_does_not_split_last_event_stream(wa: WebArenaVerified, tmp_path: Path) -> None:
+    entries = [
+        _entry(
+            method="POST",
+            url="http://localhost:9999/submit/MachineLearning",
+            post_data={
+                "submission[title]": "what is the SOTA web navigation agent repo",
+                "submission[forum]": "10037",
+            },
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
+        _entry(
+            method="POST",
+            url="http://localhost:9999/submit/MachineLearning",
+            post_data={"submission[title]": "wrong final title", "submission[forum]": "99999"},
+            status=302,
+            mime_type="application/x-www-form-urlencoded",
+        ),
+    ]
+
+    assert float(_evaluate(wa, tmp_path, 604, entries).score) == 0.0
+
+
 def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebArenaVerified, tmp_path: Path) -> None:
     config = wa.get_task(811).network_event_evaluator_cfgs[0]
     assert config.post_data_schema == {

From 099e27fe9ab2c3e5ae903346927d56f41f55a47d Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:43:12 +0800
Subject: [PATCH 5/6] Correct canonical task contracts

---
 assets/dataset/webarena-verified.json         | 123 +++++++++--------
 tests/api/test_corrected_task_contracts.py    |  24 +++-
 .../test_network_event_contracts.py           |  28 ++++
 .../test_known_contract_corrections.py        | 127 ++++++++++++++++++
 4 files changed, 236 insertions(+), 66 deletions(-)

diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json
index 3add3f0..87773b5 100644
--- a/assets/dataset/webarena-verified.json
+++ b/assets/dataset/webarena-verified.json
@@ -10821,15 +10821,17 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
-        "last_event_only": false,
+        "last_event_only": true,
+        "navigation_only": false,
         "expected": {
-          "url": "^__SHOPPING_ADMIN__/catalog/product/save/id/126/type/configurable/store/0/set/\\d+/back/edit$",
-          "post_data": {"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"},
+          "url": "^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$",
+          "http_method": "POST",
+          "post_data": {"excluded": "false", "search": "hollister", "namespace": "product_listing"},
           "response_status": 302
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["wikipedia", "map"],
@@ -14576,17 +14578,6 @@
         "results_schema": {"type": "null"},
         "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null}
       },
-      {
-        "evaluator": "NetworkEventEvaluator",
-        "expected": {
-          "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93",
-          "http_method": "POST",
-          "response_status": 302,
-          "post_data": {
-            "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Brown.*$"
-          }
-        }
-      },
       {
         "evaluator": "NetworkEventEvaluator",
         "post_data_schema": {
@@ -14606,7 +14597,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -14626,17 +14617,6 @@
         "results_schema": {"type": "null"},
         "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null}
       },
-      {
-        "evaluator": "NetworkEventEvaluator",
-        "expected": {
-          "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93",
-          "http_method": "POST",
-          "response_status": 302,
-          "post_data": {
-            "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Blue.*$"
-          }
-        }
-      },
       {
         "evaluator": "NetworkEventEvaluator",
         "post_data_schema": {
@@ -14650,13 +14630,13 @@
           "http_method": "POST",
           "response_status": 302,
           "post_data": {
-            "product[name]": "Frankie Sweatshirt",
+            "product[name]": "Frankie  Sweatshirt",
             "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes": ["size: s, color: blue", "size: m, color: blue"]
           }
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -14678,12 +14658,17 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": true,
+        "navigation_only": false,
+        "ignored_query_params": ["isAjax"],
         "expected": {
-          "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144",
+          "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$",
           "http_method": "POST",
-          "response_status": 302,
+          "response_status": 200,
           "post_data": {
-            "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXXL.*$"
+            "options[0][label]": "XXXL",
+            "options[0][attribute_id]": "144",
+            "options[0][is_new]": "true"
           }
         }
       },
@@ -14706,7 +14691,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -14728,12 +14713,17 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "last_event_only": true,
+        "navigation_only": false,
+        "ignored_query_params": ["isAjax"],
         "expected": {
-          "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144",
+          "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$",
           "http_method": "POST",
-          "response_status": 302,
+          "response_status": 200,
           "post_data": {
-            "serialized_options": "^.*&swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXS.*$"
+            "options[0][label]": "XXS",
+            "options[0][attribute_id]": "144",
+            "options[0][is_new]": "true"
           }
         }
       },
@@ -14756,7 +14746,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -15512,7 +15502,7 @@
           "properties": { "$.actions[0].content": {"type": "string", "format": "markdown"} }
         },
         "expected": {
-          "url": "__GITLAB__//api/v4/projects/byteblaze%2Fdo-it-myself-with/repository/commits",
+          "url": "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits",
           "http_method": "POST",
           "post_data": {
             "branch": "main",
@@ -15525,7 +15515,7 @@
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -19101,6 +19091,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "expected": {
           "url": "__SHOPPING_ADMIN__/mui/index/render/",
           "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"},
@@ -19115,7 +19106,7 @@
         "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"]
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19133,6 +19124,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "expected": {
           "url": "__SHOPPING_ADMIN__/mui/index/render/",
           "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"},
@@ -19147,7 +19139,7 @@
         "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"]
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19165,6 +19157,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "expected": {
           "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$",
           "query_params": {
@@ -19180,7 +19173,7 @@
         "decode_base64_query": true
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["shopping_admin"],
@@ -19198,6 +19191,7 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "expected": {
           "url": "__SHOPPING_ADMIN__/mui/index/render/",
           "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"},
@@ -19212,7 +19206,7 @@
         "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"]
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["reddit", "gitlab"],
@@ -19872,13 +19866,15 @@
             "name": "spring sale",
             "website_ids[0]": "1",
             "customer_group_ids[0]": "1",
+            "customer_group_ids[1]": "2",
+            "customer_group_ids[2]": "3",
             "simple_action": "by_percent",
             "discount_amount": 20
           }
         }
       }
     ],
-    "revision": 3
+    "revision": 4
   },
   {
     "sites": ["shopping_admin"],
@@ -19904,13 +19900,15 @@
             "name": "fall discount",
             "website_ids[0]": "1",
             "customer_group_ids[0]": "1",
+            "customer_group_ids[1]": "2",
+            "customer_group_ids[2]": "3",
             "simple_action": "cart_fixed",
             "discount_amount": 10
           }
         }
       }
     ],
-    "revision": 3
+    "revision": 4
   },
   {
     "sites": ["shopping_admin"],
@@ -21384,11 +21382,12 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-75.1718916,39.9011873;-79.9427192,40.4441897.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1718916,39.9011873.*$"}
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["wikipedia", "map"],
@@ -21410,11 +21409,12 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-75.1712951,39.9042046;-79.9427192,40.4441897.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1712951,39.9042046.*$"}
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["wikipedia", "map"],
@@ -21436,11 +21436,12 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-73.9265212,40.8295828;-79.9427192,40.4441897.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9265212,40.8295828.*$"}
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["wikipedia", "map"],
@@ -21462,11 +21463,12 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-73.9935443,40.7505085;-79.9427192,40.4441897.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9935443,40.7505085.*$"}
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["wikipedia", "map"],
@@ -21488,11 +21490,12 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-71.0621475,42.3662922;-79.9427192,40.4441897.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-71.0621475,42.3662922.*$"}
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["gitlab"],
@@ -22238,14 +22241,15 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/v1/.*/-71.060511,42.3554334;-1.4869496,55.0252998.*$",
+          "url": "^.*/route/v1/.*/-1.4869496,55.0252998;-71.060511,42.3554334.*$",
           "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"}
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["map", "shopping_admin"],
@@ -22267,14 +22271,15 @@
       },
       {
         "evaluator": "NetworkEventEvaluator",
+        "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.4041622,40.0757384.*$",
+          "url": "^.*/route/v1/.*/-74.4041622,40.0757384;-75.4716115,40.6022552.*$",
           "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"}
         }
       }
     ],
-    "revision": 2
+    "revision": 3
   },
   {
     "sites": ["map"],
diff --git a/tests/api/test_corrected_task_contracts.py b/tests/api/test_corrected_task_contracts.py
index d096350..3e59d39 100644
--- a/tests/api/test_corrected_task_contracts.py
+++ b/tests/api/test_corrected_task_contracts.py
@@ -7,8 +7,9 @@
 import pytest
 
 from webarena_verified.api import WebArenaVerified
-from webarena_verified.types.config import WebArenaVerifiedConfig
+from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig
 from webarena_verified.types.eval import EvalStatus
+from webarena_verified.types.task import WebArenaSite
 
 
 def _event(url, *, method="GET", form=None, referer=None, navigation=False):
@@ -48,8 +49,8 @@ def _evaluate(tmp_path, task, entries, task_type):
         config=WebArenaVerifiedConfig(
             test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json",
             environments={
-                "__GITLAB__": {"urls": ["http://localhost:8023"]},
-                "__SHOPPING_ADMIN__": {"urls": ["http://localhost:7780/admin"]},
+                WebArenaSite.GITLAB: EnvironmentConfig(urls=["http://localhost:8023"]),
+                WebArenaSite.SHOPPING_ADMIN: EnvironmentConfig(urls=["http://localhost:7780/admin"]),
             },
         )
     )
@@ -104,14 +105,23 @@ def test_empty_project_accepts_unchecked_but_rejects_enabled_readme(tmp_path, re
         (700, "fall discount", "cart_fixed", 10),
     ],
 )
-@pytest.mark.parametrize("customer_group", ["1", "2"])
-def test_price_rule_form_requires_the_requested_customer_group(tmp_path, task, name, action, amount, customer_group):
+@pytest.mark.parametrize(
+    ("customer_groups", "expected"),
+    [
+        (("1", "2", "3"), True),
+        (("1", "2"), False),
+        (("1", "2", "4"), False),
+    ],
+)
+def test_price_rule_form_requires_all_registered_customer_groups(
+    tmp_path, task, name, action, amount, customer_groups, expected
+):
     form = {
         "name": name,
         "website_ids[0]": "1",
-        "customer_group_ids[0]": customer_group,
         "simple_action": action,
         "discount_amount": amount,
     }
+    form.update({f"customer_group_ids[{index}]": value for index, value in enumerate(customer_groups)})
     entries = [_event("http://localhost:7780/admin/sales_rule/promo_quote/save/", method="POST", form=form)]
-    assert _evaluate(tmp_path, task, entries, "MUTATE") is (customer_group == "1")
+    assert _evaluate(tmp_path, task, entries, "MUTATE") is expected
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
index cd6934c..97428ca 100644
--- a/tests/core/evaluation/evaluators/test_network_event_contracts.py
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -80,6 +80,34 @@ def test_repeated_endpoint_contracts_match_distinct_request_bodies(wa: WebArenaV
     assert float(_evaluate(wa, tmp_path, 567, entries[1:]).score) == 0.0
 
 
+def test_hollister_mass_status_uses_the_native_filtered_grid_request(wa: WebArenaVerified, tmp_path: Path) -> None:
+    correct = _entry(
+        method="POST",
+        url="http://localhost:7780/admin/catalog/product/massStatus/status/1/",
+        post_data={"excluded": "false", "search": "hollister", "namespace": "product_listing"},
+        status=302,
+        mime_type="application/x-www-form-urlencoded",
+    )
+    unrelated_upstream_contract = _entry(
+        method="GET",
+        url="http://localhost:7780/admin/catalog/product/save/id/126/type/configurable/store/0/set/4/back/edit",
+        post_data={"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"},
+        status=302,
+        mime_type="application/x-www-form-urlencoded",
+    )
+    later_wrong_mass_status = _entry(
+        method="POST",
+        url="http://localhost:7780/admin/catalog/product/massStatus/status/1/",
+        post_data={"excluded": "false", "search": "nike", "namespace": "product_listing"},
+        status=302,
+        mime_type="application/x-www-form-urlencoded",
+    )
+
+    assert float(_evaluate(wa, tmp_path, 423, [correct]).score) == 1.0
+    assert float(_evaluate(wa, tmp_path, 423, [unrelated_upstream_contract]).score) == 0.0
+    assert float(_evaluate(wa, tmp_path, 423, [correct, later_wrong_mass_status]).score) == 0.0
+
+
 @pytest.mark.parametrize("task_id", [742, 743, 745, 746])
 def test_project_creation_with_multiple_members_matches_complete_trace(
     wa: WebArenaVerified, tmp_path: Path, task_id: int
diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py
index f50da5a..a6cda0a 100644
--- a/tests/dataset/test_known_contract_corrections.py
+++ b/tests/dataset/test_known_contract_corrections.py
@@ -40,6 +40,133 @@ def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives(
         assert "website_ids" not in post_data
         assert "customer_group_ids" not in post_data
 
+    for task_id in (699, 700):
+        task = dataset_by_task_id[task_id]
+        post_data = _network_expectations(task)[0]["post_data"]
+        assert task["revision"] == 4
+        assert [post_data[f"customer_group_ids[{index}]"] for index in range(3)] == [
+            "1",
+            "2",
+            "3",
+        ]
+
+
+def test_mass_status_contract_targets_the_filtered_magento_mutation(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    task = dataset_by_task_id[423]
+    evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
+    assert task["revision"] == 3
+    assert evaluator["last_event_only"] is True
+    assert evaluator["navigation_only"] is False
+    assert evaluator["expected"] == {
+        "url": r"^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$",
+        "http_method": "POST",
+        "post_data": {
+            "excluded": "false",
+            "search": "hollister",
+            "namespace": "product_listing",
+        },
+        "response_status": 302,
+    }
+
+
+def test_existing_color_options_require_only_the_new_product_variations(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    expected = {
+        547: ("Phoebe Zipper Sweatshirt", ["size: s, color: brown"]),
+        548: ("Frankie  Sweatshirt", ["size: s, color: blue", "size: m, color: blue"]),
+    }
+    matrix_key = "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes"
+    for task_id, (product_name, variations) in expected.items():
+        task = dataset_by_task_id[task_id]
+        evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"]
+        assert task["revision"] == 3
+        assert len(evaluators) == 1
+        assert "product_attribute/save" not in evaluators[0]["expected"]["url"]
+        assert evaluators[0]["expected"]["post_data"]["product[name]"] == product_name
+        assert evaluators[0]["expected"]["post_data"][matrix_key] == variations
+
+
+def test_new_size_options_use_the_native_configurable_product_endpoint(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    for task_id, label in ((549, "XXXL"), (550, "XXS")):
+        task = dataset_by_task_id[task_id]
+        evaluator = next(
+            item
+            for item in task["eval"]
+            if item["evaluator"] == "NetworkEventEvaluator" and "createOptions" in item["expected"]["url"]
+        )
+        assert task["revision"] == 3
+        assert evaluator["last_event_only"] is True
+        assert evaluator["navigation_only"] is False
+        assert evaluator["ignored_query_params"] == ["isAjax"]
+        assert evaluator["expected"] == {
+            "url": r"^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$",
+            "http_method": "POST",
+            "response_status": 200,
+            "post_data": {
+                "options[0][label]": label,
+                "options[0][attribute_id]": "144",
+                "options[0][is_new]": "true",
+            },
+        }
+
+
+def test_readme_commit_targets_the_created_repository(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    task = dataset_by_task_id[566]
+    commit = _network_expectations(task)[1]
+    assert task["revision"] == 3
+    assert commit["url"] == "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits"
+
+
+def test_order_grid_status_contracts_evaluate_the_xhr(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    for task_id in (677, 678, 679, 680):
+        task = dataset_by_task_id[task_id]
+        evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
+        assert task["revision"] == 3
+        assert evaluator["navigation_only"] is False
+        assert "mui/index/render/" in evaluator["expected"]["url"]
+
+
+def test_route_contracts_follow_the_intent_origin_to_destination(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    coordinates = {
+        737: "-79.9427192,40.4441897;-75.1718916,39.9011873",
+        738: "-79.9427192,40.4441897;-75.1712951,39.9042046",
+        739: "-79.9427192,40.4441897;-73.9265212,40.8295828",
+        740: "-79.9427192,40.4441897;-73.9935443,40.7505085",
+        741: "-79.9427192,40.4441897;-71.0621475,42.3662922",
+        759: "-1.4869496,55.0252998;-71.060511,42.3554334",
+        760: "-74.4041622,40.0757384;-75.4716115,40.6022552",
+    }
+    for task_id, coordinate_pair in coordinates.items():
+        task = dataset_by_task_id[task_id]
+        evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
+        assert task["revision"] == 3
+        assert evaluator["navigation_only"] is False
+        assert coordinate_pair in evaluator["expected"]["url"]
+
+
+def test_replay_specific_changes_do_not_weaken_canonical_contracts(
+    dataset_by_task_id: dict[int, dict[str, Any]],
+) -> None:
+    for task_id in (448, 744):
+        task = dataset_by_task_id[task_id]
+        evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"]
+        assert task["revision"] == 2
+        assert all(item.get("last_event_only", True) is True for item in evaluators)
+
+    cart_totals = _network_expectations(dataset_by_task_id[510])[1]
+    assert cart_totals["response_content"]["items_qty"] == 1
+
 
 def test_repeated_member_requests_explicitly_match_any_event(
     dataset_by_task_id: dict[int, dict[str, Any]],

From 361d11ce6042afdda13b32ded463942b81e0cd61 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 18:50:11 +0800
Subject: [PATCH 6/6] Correct route contract direction

---
 assets/dataset/webarena-verified.json         | 24 ++---
 .../test_network_event_contracts.py           | 87 ++++++++++++++++++-
 .../test_known_contract_corrections.py        | 13 ++-
 3 files changed, 105 insertions(+), 19 deletions(-)

diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json
index 87773b5..fb67b2f 100644
--- a/assets/dataset/webarena-verified.json
+++ b/assets/dataset/webarena-verified.json
@@ -6722,10 +6722,10 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-71.0579762,42.3603713.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-71.0579762,42.3603713;-68.2177005,44.3494709.*$"}
       }
     ],
-    "revision": 4
+    "revision": 5
   },
   {
     "sites": ["wikipedia", "map"],
@@ -6762,10 +6762,10 @@
       {
         "evaluator": "NetworkEventEvaluator",
         "ignored_query_params_patterns": [".*"],
-        "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-70.2545299,43.6599147.*$"}
+        "expected": {"url": "^.*/route/v1/.*/-70.2545299,43.6599147;-68.2177005,44.3494709.*$"}
       }
     ],
-    "revision": 4
+    "revision": 5
   },
   {
     "sites": ["wikipedia", "map"],
@@ -6803,12 +6803,12 @@
         "evaluator": "NetworkEventEvaluator",
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/.*/-68.2177005,44.3494709;-68.767507,44.8030715.*$",
+          "url": "^.*/route/.*/-68.767507,44.8030715;-68.2177005,44.3494709.*$",
           "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"}
         }
       }
     ],
-    "revision": 4
+    "revision": 5
   },
   {
     "sites": ["wikipedia", "map"],
@@ -6846,12 +6846,12 @@
         "evaluator": "NetworkEventEvaluator",
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/v1/.*/-68.2177005,44.3494709;-68.8315387,44.0478975.*$",
+          "url": "^.*/route/v1/.*/-68.8315387,44.0478975;-68.2177005,44.3494709.*$",
           "headers": {"Cookie": "^.*_osm_directions_engine=fossgis_osrm_bicycle.*$"}
         }
       }
     ],
-    "revision": 4
+    "revision": 5
   },
   {
     "sites": ["shopping"],
@@ -22244,12 +22244,12 @@
         "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/v1/.*/-1.4869496,55.0252998;-71.060511,42.3554334.*$",
+          "url": "^.*/route/v1/.*/-71.060511,42.3554334;-74.0060152,40.7127281.*$",
           "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"}
         }
       }
     ],
-    "revision": 3
+    "revision": 4
   },
   {
     "sites": ["map", "shopping_admin"],
@@ -22274,12 +22274,12 @@
         "navigation_only": false,
         "ignored_query_params_patterns": [".*"],
         "expected": {
-          "url": "^.*/route/v1/.*/-74.4041622,40.0757384;-75.4716115,40.6022552.*$",
+          "url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.4041622,40.0757384.*$",
           "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"}
         }
       }
     ],
-    "revision": 3
+    "revision": 4
   },
   {
     "sites": ["map"],
diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py
index 97428ca..8922096 100644
--- a/tests/core/evaluation/evaluators/test_network_event_contracts.py
+++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py
@@ -17,6 +17,7 @@ def _entry(
     post_data: dict[str, Any],
     status: int,
     mime_type: str = "application/json",
+    request_headers: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     body = json.dumps(post_data) if mime_type == "application/json" else urlencode(post_data, doseq=True)
     return {
@@ -25,7 +26,7 @@ def _entry(
         "request": {
             "method": method,
             "url": url,
-            "headers": [],
+            "headers": [{"name": name, "value": value} for name, value in (request_headers or {}).items()],
             "cookies": [],
             "queryString": [],
             "postData": {"mimeType": mime_type, "text": body},
@@ -42,7 +43,13 @@ def _entry(
     }
 
 
-def _evaluate(wa: WebArenaVerified, tmp_path: Path, task_id: int, entries: list[dict[str, Any]]):
+def _evaluate(
+    wa: WebArenaVerified,
+    tmp_path: Path,
+    task_id: int,
+    entries: list[dict[str, Any]],
+    agent_response: dict[str, Any] | None = None,
+):
     trace = tmp_path / f"task-{task_id}.har"
     trace.write_text(
         json.dumps(
@@ -57,7 +64,7 @@ def _evaluate(wa: WebArenaVerified, tmp_path: Path, task_id: int, entries: list[
     )
     return wa.evaluate_task(
         task_id=task_id,
-        agent_response={"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
+        agent_response=agent_response or {"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None},
         network_trace=trace,
     )
 
@@ -108,6 +115,80 @@ def test_hollister_mass_status_uses_the_native_filtered_grid_request(wa: WebAren
     assert float(_evaluate(wa, tmp_path, 423, [correct, later_wrong_mass_status]).score) == 0.0
 
 
+@pytest.mark.parametrize(
+    ("task_id", "origin", "destination", "cookie"),
+    [
+        (265, "-71.0579762,42.3603713", "-68.2177005,44.3494709", ""),
+        (266, "-70.2545299,43.6599147", "-68.2177005,44.3494709", ""),
+        (267, "-68.767507,44.8030715", "-68.2177005,44.3494709", ""),
+        (
+            268,
+            "-68.8315387,44.0478975",
+            "-68.2177005,44.3494709",
+            "_osm_directions_engine=fossgis_osrm_bicycle",
+        ),
+        (759, "-71.060511,42.3554334", "-74.0060152,40.7127281", ""),
+        (760, "-75.4716115,40.6022552", "-74.4041622,40.0757384", ""),
+    ],
+)
+def test_route_contracts_require_intent_direction(
+    wa: WebArenaVerified,
+    tmp_path: Path,
+    task_id: int,
+    origin: str,
+    destination: str,
+    cookie: str,
+) -> None:
+    expected_agent_response = wa.get_task(task_id).expected_agent_response.model_dump(mode="json")
+
+    def route_entry(first: str, second: str) -> dict[str, Any]:
+        return _entry(
+            method="GET",
+            url=f"http://localhost:3000/osrm/routed/route/v1/driving/{first};{second}?steps=true",
+            post_data={},
+            status=200,
+            request_headers={"Cookie": cookie},
+        )
+
+    def route_trace(first: str, second: str) -> list[dict[str, Any]]:
+        route = route_entry(first, second)
+        if not wa.get_task(task_id).is_navigate_task:
+            return [route]
+        navigation = _entry(
+            method="GET",
+            url="http://localhost:3000/directions",
+            post_data={},
+            status=200,
+            request_headers={"Accept": "text/html", "Sec-Fetch-Dest": "document"},
+        )
+        return [navigation, route]
+
+    assert (
+        float(
+            _evaluate(
+                wa,
+                tmp_path,
+                task_id,
+                route_trace(origin, destination),
+                expected_agent_response,
+            ).score
+        )
+        == 1.0
+    )
+    assert (
+        float(
+            _evaluate(
+                wa,
+                tmp_path,
+                task_id,
+                route_trace(destination, origin),
+                expected_agent_response,
+            ).score
+        )
+        == 0.0
+    )
+
+
 @pytest.mark.parametrize("task_id", [742, 743, 745, 746])
 def test_project_creation_with_multiple_members_matches_complete_trace(
     wa: WebArenaVerified, tmp_path: Path, task_id: int
diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py
index a6cda0a..97a7844 100644
--- a/tests/dataset/test_known_contract_corrections.py
+++ b/tests/dataset/test_known_contract_corrections.py
@@ -139,19 +139,24 @@ def test_route_contracts_follow_the_intent_origin_to_destination(
     dataset_by_task_id: dict[int, dict[str, Any]],
 ) -> None:
     coordinates = {
+        265: "-71.0579762,42.3603713;-68.2177005,44.3494709",
+        266: "-70.2545299,43.6599147;-68.2177005,44.3494709",
+        267: "-68.767507,44.8030715;-68.2177005,44.3494709",
+        268: "-68.8315387,44.0478975;-68.2177005,44.3494709",
         737: "-79.9427192,40.4441897;-75.1718916,39.9011873",
         738: "-79.9427192,40.4441897;-75.1712951,39.9042046",
         739: "-79.9427192,40.4441897;-73.9265212,40.8295828",
         740: "-79.9427192,40.4441897;-73.9935443,40.7505085",
         741: "-79.9427192,40.4441897;-71.0621475,42.3662922",
-        759: "-1.4869496,55.0252998;-71.060511,42.3554334",
-        760: "-74.4041622,40.0757384;-75.4716115,40.6022552",
+        759: "-71.060511,42.3554334;-74.0060152,40.7127281",
+        760: "-75.4716115,40.6022552;-74.4041622,40.0757384",
     }
     for task_id, coordinate_pair in coordinates.items():
         task = dataset_by_task_id[task_id]
         evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator")
-        assert task["revision"] == 3
-        assert evaluator["navigation_only"] is False
+        assert task["revision"] == (5 if task_id in {265, 266, 267, 268} else 4 if task_id in {759, 760} else 3)
+        if task_id not in {265, 266, 267, 268}:
+            assert evaluator["navigation_only"] is False
         assert coordinate_pair in evaluator["expected"]["url"]