diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json index fe1af37..fb67b2f 100644 --- a/assets/dataset/webarena-verified.json +++ b/assets/dataset/webarena-verified.json @@ -6722,10 +6722,10 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-71.0579762,42.3603713.*$"} + "expected": {"url": "^.*/route/v1/.*/-71.0579762,42.3603713;-68.2177005,44.3494709.*$"} } ], - "revision": 4 + "revision": 5 }, { "sites": ["wikipedia", "map"], @@ -6762,10 +6762,10 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-70.2545299,43.6599147.*$"} + "expected": {"url": "^.*/route/v1/.*/-70.2545299,43.6599147;-68.2177005,44.3494709.*$"} } ], - "revision": 4 + "revision": 5 }, { "sites": ["wikipedia", "map"], @@ -6803,12 +6803,12 @@ "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/.*/-68.2177005,44.3494709;-68.767507,44.8030715.*$", + "url": "^.*/route/.*/-68.767507,44.8030715;-68.2177005,44.3494709.*$", "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"} } } ], - "revision": 4 + "revision": 5 }, { "sites": ["wikipedia", "map"], @@ -6846,12 +6846,12 @@ "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/v1/.*/-68.2177005,44.3494709;-68.8315387,44.0478975.*$", + "url": "^.*/route/v1/.*/-68.8315387,44.0478975;-68.2177005,44.3494709.*$", "headers": {"Cookie": "^.*_osm_directions_engine=fossgis_osrm_bicycle.*$"} } } ], - "revision": 4 + "revision": 5 }, { "sites": ["shopping"], @@ -10821,15 +10821,17 @@ }, { "evaluator": "NetworkEventEvaluator", - "last_event_only": false, + "last_event_only": true, + "navigation_only": false, "expected": { - "url": "^__SHOPPING_ADMIN__/catalog/product/save/id/126/type/configurable/store/0/set/\\d+/back/edit$", - "post_data": {"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"}, + "url": "^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$", + "http_method": "POST", + "post_data": {"excluded": "false", "search": "hollister", "namespace": "product_listing"}, "response_status": 302 } } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -14576,17 +14578,6 @@ "results_schema": {"type": "null"}, "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null} }, - { - "evaluator": "NetworkEventEvaluator", - "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93", - "http_method": "POST", - "response_status": 302, - "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Brown.*$" - } - } - }, { "evaluator": "NetworkEventEvaluator", "post_data_schema": { @@ -14606,7 +14597,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14626,17 +14617,6 @@ "results_schema": {"type": "null"}, "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null} }, - { - "evaluator": "NetworkEventEvaluator", - "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93", - "http_method": "POST", - "response_status": 302, - "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Blue.*$" - } - } - }, { "evaluator": "NetworkEventEvaluator", "post_data_schema": { @@ -14650,13 +14630,13 @@ "http_method": "POST", "response_status": 302, "post_data": { - "product[name]": "Frankie Sweatshirt", + "product[name]": "Frankie Sweatshirt", "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes": ["size: s, color: blue", "size: m, color: blue"] } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14678,12 +14658,17 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": true, + "navigation_only": false, + "ignored_query_params": ["isAjax"], "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144", + "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", "http_method": "POST", - "response_status": 302, + "response_status": 200, "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXXL.*$" + "options[0][label]": "XXXL", + "options[0][attribute_id]": "144", + "options[0][is_new]": "true" } } }, @@ -14706,7 +14691,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14728,12 +14713,17 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": true, + "navigation_only": false, + "ignored_query_params": ["isAjax"], "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144", + "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", "http_method": "POST", - "response_status": 302, + "response_status": 200, "post_data": { - "serialized_options": "^.*&swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXS.*$" + "options[0][label]": "XXS", + "options[0][attribute_id]": "144", + "options[0][is_new]": "true" } } }, @@ -14756,7 +14746,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -15512,7 +15502,7 @@ "properties": { "$.actions[0].content": {"type": "string", "format": "markdown"} } }, "expected": { - "url": "__GITLAB__//api/v4/projects/byteblaze%2Fdo-it-myself-with/repository/commits", + "url": "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits", "http_method": "POST", "post_data": { "branch": "main", @@ -15525,7 +15515,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15546,6 +15536,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/184/members", "http_method": "POST", @@ -15555,6 +15546,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/184/members", "http_method": "POST", @@ -15563,7 +15555,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15581,6 +15573,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/174/members", "http_method": "POST", @@ -15590,6 +15583,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/174/members", "http_method": "POST", @@ -15598,7 +15592,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15619,6 +15613,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/185/members", "http_method": "POST", @@ -15628,6 +15623,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/185/members", "http_method": "POST", @@ -15636,7 +15632,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15657,6 +15653,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15666,6 +15663,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15675,6 +15673,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15684,6 +15683,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15692,7 +15692,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping"], @@ -19091,6 +19091,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19105,7 +19106,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19123,6 +19124,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19137,7 +19139,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19155,6 +19157,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$", "query_params": { @@ -19170,7 +19173,7 @@ "decode_base64_query": true } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19188,6 +19191,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19202,7 +19206,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit", "gitlab"], @@ -19862,13 +19866,15 @@ "name": "spring sale", "website_ids[0]": "1", "customer_group_ids[0]": "1", + "customer_group_ids[1]": "2", + "customer_group_ids[2]": "3", "simple_action": "by_percent", "discount_amount": 20 } } } ], - "revision": 3 + "revision": 4 }, { "sites": ["shopping_admin"], @@ -19894,13 +19900,15 @@ "name": "fall discount", "website_ids[0]": "1", "customer_group_ids[0]": "1", + "customer_group_ids[1]": "2", + "customer_group_ids[2]": "3", "simple_action": "cart_fixed", "discount_amount": 10 } } } ], - "revision": 3 + "revision": 4 }, { "sites": ["shopping_admin"], @@ -21374,11 +21382,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-75.1718916,39.9011873;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1718916,39.9011873.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21400,11 +21409,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-75.1712951,39.9042046;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1712951,39.9042046.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21426,11 +21436,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-73.9265212,40.8295828;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9265212,40.8295828.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21452,11 +21463,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-73.9935443,40.7505085;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9935443,40.7505085.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21478,11 +21490,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-71.0621475,42.3662922;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-71.0621475,42.3662922.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -21513,6 +21526,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21522,6 +21536,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21530,7 +21545,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -21561,6 +21576,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21570,6 +21586,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21578,7 +21595,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -21648,6 +21665,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21657,6 +21675,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21666,6 +21685,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21674,7 +21694,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -21705,6 +21725,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21714,6 +21735,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21723,6 +21745,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "^__GITLAB__/api/v4/projects/\\d+/members$", "http_method": "POST", @@ -21731,7 +21754,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -22218,14 +22241,15 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/v1/.*/-71.060511,42.3554334;-1.4869496,55.0252998.*$", + "url": "^.*/route/v1/.*/-71.060511,42.3554334;-74.0060152,40.7127281.*$", "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["map", "shopping_admin"], @@ -22247,6 +22271,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], "expected": { "url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.4041622,40.0757384.*$", @@ -22254,7 +22279,7 @@ } } ], - "revision": 2 + "revision": 4 }, { "sites": ["map"], @@ -23951,6 +23976,12 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params": ["serializer"], + "post_data_schema": { + "type": "object", + "properties": { + "$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}} + } + }, "expected": { "url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1478.json", "http_method": "PUT", @@ -23958,6 +23989,6 @@ } } ], - "revision": 2 + "revision": 3 } ] diff --git a/src/webarena_verified/core/evaluation/data_types/base.py b/src/webarena_verified/core/evaluation/data_types/base.py index dc97022..ef34ec6 100644 --- a/src/webarena_verified/core/evaluation/data_types/base.py +++ b/src/webarena_verified/core/evaluation/data_types/base.py @@ -244,10 +244,7 @@ def _normalize_string(self, s: str) -> str: # Step 0: Try to derender URLs (best effort, no exceptions) # Let the derender_url_fct decide whether to process the string - if self.derender_url_fct is not None and "http" in s: - with contextlib.suppress(Exception): - s = self.derender_url_fct(s, strict=False) - # If derendering fails for any reason, continue with original string + s = self._derender_urls(s) # Step 1: Remove common non-semantic marks BEFORE NFKC normalization # U+2122 ™, U+00AE ®, U+2120 ℠, U+00A9 © @@ -286,3 +283,13 @@ def _normalize_string(self, s: str) -> str: # Step 5: Casefold for robust case-insensitive comparison return s.casefold() + + def _derender_urls(self, value: str) -> str: + """Replace deployment URLs through the configured best-effort callback.""" + if self.derender_url_fct is None or "http" not in value: + return value + with contextlib.suppress(Exception): + # Evaluators bind the desired strictness when they construct this + # callback. Passing it again breaks functools.partial callbacks. + return self.derender_url_fct(value) + return value diff --git a/src/webarena_verified/core/evaluation/data_types/markdown_string.py b/src/webarena_verified/core/evaluation/data_types/markdown_string.py index 34a0054..1ce4f73 100644 --- a/src/webarena_verified/core/evaluation/data_types/markdown_string.py +++ b/src/webarena_verified/core/evaluation/data_types/markdown_string.py @@ -40,7 +40,7 @@ def _normalize_string(self, s: str) -> str: We only strip leading/trailing whitespace. """ - return s.strip() + return self._derender_urls(s).strip() def _type_normalize(self, value: Any) -> str: """Normalize markdown string for comparison. diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py index 32c17c2..4ad252a 100644 --- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py +++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py @@ -5,6 +5,7 @@ """ import re +from collections.abc import Mapping from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, cast @@ -13,7 +14,7 @@ from webarena_verified.core.evaluation.data_types import URL from webarena_verified.core.utils import logger from webarena_verified.core.utils.jsonpath_utils import extract_jsonpath_value, is_jsonpath_key -from webarena_verified.types.eval import EvalAssertion, EvalStatus, TaskEvalContext +from webarena_verified.types.eval import EvalAssertion, EvalStatus, EvaluatorResult, TaskEvalContext from webarena_verified.types.task import NetworkEventEvaluatorCfg, NetworkEventSpec from webarena_verified.types.tracing import NetworkEvent @@ -23,6 +24,67 @@ from webarena_verified.types.common import SerializableMappingProxyType +_PLACEHOLDER = re.compile(r"\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}") + + +def _template_match(template: str, actual: Any, bindings: dict[str, str]) -> bool: + """Match a dynamic evaluator template and consistently bind its names.""" + if not isinstance(actual, (str, int, float)): + return False + cursor = 0 + parts = ["^"] + local_groups: set[str] = set() + for match in _PLACEHOLDER.finditer(template): + parts.append(re.escape(template[cursor : match.start()])) + name = match.group(1) + if name in bindings: + parts.append(re.escape(bindings[name])) + elif name in local_groups: + parts.append(f"(?P={name})") + else: + parts.append(f"(?P<{name}>[^/?#&]+)") + local_groups.add(name) + cursor = match.end() + parts.extend((re.escape(template[cursor:]), "$")) + matched = re.fullmatch("".join(parts), str(actual)) + if matched is None: + return False + bindings.update(matched.groupdict()) + return True + + +def _placeholder_names(value: Any) -> set[str]: + if isinstance(value, str): + return set(_PLACEHOLDER.findall(value)) + if isinstance(value, Mapping): + names: set[str] = set() + for key, item in value.items(): + names.update(_placeholder_names(key)) + names.update(_placeholder_names(item)) + return names + if isinstance(value, (list, tuple)): + names = set() + for item in value: + names.update(_placeholder_names(item)) + return names + return set() + + +def _substitute_placeholders(value: Any, bindings: Mapping[str, str]) -> Any: + if isinstance(value, str): + return _PLACEHOLDER.sub(lambda match: bindings[match.group(1)], value) + if isinstance(value, list): + return [_substitute_placeholders(item, bindings) for item in value] + if isinstance(value, tuple): + return tuple(_substitute_placeholders(item, bindings) for item in value) + if isinstance(value, Mapping): + return { + _substitute_placeholders(key, bindings): _substitute_placeholders(item, bindings) + for key, item in value.items() + } + return value + + class NetworkEventEvaluator(BaseEvaluator[NetworkEventEvaluatorCfg]): """Validates network events using four-step architecture. @@ -256,6 +318,16 @@ def _normalize_post_data_field( if post_data_dict: post_data_schema = config.post_data_schema or {} + if ( + not strict + and self._contains_array_schema(post_data_schema) + and not self._matches_json_schema(post_data_dict, post_data_schema) + ): + # Preserve the mismatched raw value so structural comparison + # rejects it instead of coercing (for example, "2330" to 2330). + normalized["post_data"] = post_data_dict + return + _derender_url_fct = partial(context.config.derender_url, sites=context.task.sites, strict=strict) normalized["post_data"] = self.value_normalizer.normalize( post_data_dict, post_data_schema, strict=strict, derender_url_fct=_derender_url_fct @@ -695,3 +767,221 @@ def _should_ignore_post_param(self, param_name: str, config: NetworkEventEvaluat continue return False + + def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg) -> EvaluatorResult: + """Evaluate each runtime-bound form of a dynamic network contract.""" + candidates = self._runtime_configs(context, config) + if not candidates: + return super().evaluate(context=context, config=config) + + first_result: EvaluatorResult | None = None + for candidate, source_events in candidates: + candidate_context = context + if source_events is not None: + candidate_context = context.model_copy( + update={"network_trace": context.network_trace.model_copy(update={"events": source_events})} + ) + result = super().evaluate(context=candidate_context, config=candidate) + first_result = first_result or result + if result.score == 1: + return result + assert first_result is not None + return first_result + + def _runtime_configs( + self, context: TaskEvalContext, config: NetworkEventEvaluatorCfg + ) -> list[tuple[NetworkEventEvaluatorCfg, tuple[NetworkEvent, ...] | None]]: + """Bind placeholders and group ordered events by resolved binding.""" + expected = config.expected.model_dump(mode="python", exclude_none=False) + names = _placeholder_names(expected) + candidates_by_binding: dict[ + tuple[tuple[str, str], ...], tuple[NetworkEventEvaluatorCfg, tuple[tuple[str, str], ...]] + ] = {} + events_by_stream: dict[tuple[tuple[str, str], ...], list[NetworkEvent]] = {} + + if names: + raw_urls = expected.get("url") + url_templates = list(raw_urls) if isinstance(raw_urls, (list, tuple)) else [raw_urls] + post_templates = expected.get("post_data") or {} + for event in context.network_trace.evaluation_events: + expected_method = expected.get("http_method") + if expected_method and event.http_method.lower() != expected_method.lower(): + continue + + stream = self._resolve_event_stream( + event=event, + context=context, + config=config, + raw_urls=raw_urls, + url_templates=url_templates, + ) + if stream is None: + continue + stream_key, bindings = stream + events_by_stream.setdefault(stream_key, []).append(event) + + bindings = self._bind_post_placeholders(post_templates, event.post_data or {}, bindings) + if bindings is not None and set(bindings) == names: + corrected_expected = _substitute_placeholders(expected, bindings) + binding_key = tuple(sorted(bindings.items())) + if binding_key not in candidates_by_binding: + candidates_by_binding[binding_key] = ( + config.model_copy( + update={"expected": config.expected.model_copy(update=corrected_expected)} + ), + stream_key, + ) + + candidates = [ + (candidate, tuple(events_by_stream[stream_key])) + for candidate, stream_key in candidates_by_binding.values() + ] + else: + candidates = [(config, None)] + + return [(self._with_inferred_post_schema(candidate), source_event) for candidate, source_event in candidates] + + def _resolve_event_stream( + self, + *, + event: NetworkEvent, + context: TaskEvalContext, + config: NetworkEventEvaluatorCfg, + raw_urls: Any, + url_templates: list[Any], + ) -> tuple[tuple[tuple[str, str], ...], dict[str, str]] | None: + """Resolve URL bindings and apply the evaluator's normal prefilter.""" + bindings: dict[str, str] = {} + dynamic_urls = [item for item in url_templates if isinstance(item, str) and _PLACEHOLDER.search(item)] + resolved_urls = raw_urls + if dynamic_urls: + actual_url = context.config.derender_url(event.url, sites=context.task.sites, strict=False) + for item in dynamic_urls: + candidate_bindings = bindings.copy() + if _template_match(item, actual_url, candidate_bindings): + bindings = candidate_bindings + resolved_urls = _substitute_placeholders(item, bindings) + break + else: + return None + + filter_config = config.model_copy( + update={"expected": config.expected.model_copy(update={"url": resolved_urls})} + ) + if not self._filter_events_by_criteria((event,), context, filter_config): + return None + return tuple(sorted(bindings.items())), bindings + + @staticmethod + def _bind_post_placeholders( + post_templates: Mapping[str, Any], actual_post: Mapping[str, Any], initial: Mapping[str, str] + ) -> dict[str, str] | None: + """Bind dynamic POST keys and values without defining an event stream.""" + bindings = dict(initial) + for key_template, value_template in post_templates.items(): + actual_value = actual_post.get(key_template) + if _PLACEHOLDER.search(key_template): + matched_key = next( + (key for key in actual_post if _template_match(key_template, key, bindings.copy())), + None, + ) + if matched_key is None or not _template_match(key_template, matched_key, bindings): + return None + actual_value = actual_post[matched_key] + if ( + isinstance(value_template, str) + and _PLACEHOLDER.search(value_template) + and not _template_match(value_template, actual_value, bindings) + ): + return None + return bindings + + @staticmethod + def _with_inferred_post_schema(config: NetworkEventEvaluatorCfg) -> NetworkEventEvaluatorCfg: + """Preserve singleton array semantics when the contract omits a schema.""" + post_data = config.expected.post_data or {} + singleton_arrays = { + key for key, value in post_data.items() if isinstance(value, (list, tuple)) and len(value) == 1 + } + if not singleton_arrays or config.post_data_schema: + return config + schema = { + "type": "object", + "properties": { + key: { + "type": "array", + "items": NetworkEventEvaluator._schema_for_json_value(post_data[key][0]), + } + for key in singleton_arrays + }, + } + return config.model_copy(update={"post_data_schema": schema}) + + @staticmethod + def _schema_for_json_value(value: Any) -> dict[str, Any]: + """Infer a JSON schema without coercing distinct scalar types.""" + if value is None: + schema = {"type": "null"} + elif isinstance(value, bool): + schema = {"type": "boolean"} + elif isinstance(value, int): + schema = {"type": "integer"} + elif isinstance(value, float): + schema = {"type": "number"} + elif isinstance(value, str): + schema = {"type": "string"} + elif isinstance(value, Mapping): + schema = { + "type": "object", + "properties": {key: NetworkEventEvaluator._schema_for_json_value(item) for key, item in value.items()}, + } + elif isinstance(value, (list, tuple)): + schema = { + "type": "array", + "items": NetworkEventEvaluator._schema_for_json_value(value[0]) if value else {}, + } + else: + raise TypeError(f"Cannot infer a POST data schema for {type(value).__name__}") + return schema + + @staticmethod + def _matches_json_schema(value: Any, schema: Mapping[str, Any]) -> bool: + """Check the JSON types used by inferred singleton-array schemas.""" + schema_type = schema.get("type") + if schema_type == "null": + matches = value is None + elif schema_type == "boolean": + matches = isinstance(value, bool) + elif schema_type == "integer": + matches = isinstance(value, int) and not isinstance(value, bool) + elif schema_type == "number": + matches = isinstance(value, (int, float)) and not isinstance(value, bool) + elif schema_type == "string": + matches = isinstance(value, str) + elif schema_type == "array": + item_schema = schema.get("items", {}) + matches = isinstance(value, (list, tuple)) and all( + NetworkEventEvaluator._matches_json_schema(item, item_schema) for item in value + ) + elif schema_type == "object": + if isinstance(value, Mapping): + matches = all( + key in value and NetworkEventEvaluator._matches_json_schema(value[key], property_schema) + for key, property_schema in schema.get("properties", {}).items() + ) + else: + matches = False + else: + matches = True + return matches + + @staticmethod + def _contains_array_schema(schema: Mapping[str, Any]) -> bool: + """Return whether a schema contains an array at any depth.""" + if schema.get("type") == "array": + return True + properties = schema.get("properties", {}) + return isinstance(properties, Mapping) and any( + isinstance(property_schema, Mapping) and NetworkEventEvaluator._contains_array_schema(property_schema) + for property_schema in properties.values() + ) diff --git a/tests/api/test_corrected_task_contracts.py b/tests/api/test_corrected_task_contracts.py index d096350..3e59d39 100644 --- a/tests/api/test_corrected_task_contracts.py +++ b/tests/api/test_corrected_task_contracts.py @@ -7,8 +7,9 @@ import pytest from webarena_verified.api import WebArenaVerified -from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig from webarena_verified.types.eval import EvalStatus +from webarena_verified.types.task import WebArenaSite def _event(url, *, method="GET", form=None, referer=None, navigation=False): @@ -48,8 +49,8 @@ def _evaluate(tmp_path, task, entries, task_type): config=WebArenaVerifiedConfig( test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json", environments={ - "__GITLAB__": {"urls": ["http://localhost:8023"]}, - "__SHOPPING_ADMIN__": {"urls": ["http://localhost:7780/admin"]}, + WebArenaSite.GITLAB: EnvironmentConfig(urls=["http://localhost:8023"]), + WebArenaSite.SHOPPING_ADMIN: EnvironmentConfig(urls=["http://localhost:7780/admin"]), }, ) ) @@ -104,14 +105,23 @@ def test_empty_project_accepts_unchecked_but_rejects_enabled_readme(tmp_path, re (700, "fall discount", "cart_fixed", 10), ], ) -@pytest.mark.parametrize("customer_group", ["1", "2"]) -def test_price_rule_form_requires_the_requested_customer_group(tmp_path, task, name, action, amount, customer_group): +@pytest.mark.parametrize( + ("customer_groups", "expected"), + [ + (("1", "2", "3"), True), + (("1", "2"), False), + (("1", "2", "4"), False), + ], +) +def test_price_rule_form_requires_all_registered_customer_groups( + tmp_path, task, name, action, amount, customer_groups, expected +): form = { "name": name, "website_ids[0]": "1", - "customer_group_ids[0]": customer_group, "simple_action": action, "discount_amount": amount, } + form.update({f"customer_group_ids[{index}]": value for index, value in enumerate(customer_groups)}) entries = [_event("http://localhost:7780/admin/sales_rule/promo_quote/save/", method="POST", form=form)] - assert _evaluate(tmp_path, task, entries, "MUTATE") is (customer_group == "1") + assert _evaluate(tmp_path, task, entries, "MUTATE") is expected diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py new file mode 100644 index 0000000..8922096 --- /dev/null +++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py @@ -0,0 +1,372 @@ +"""Regression coverage for network contracts that depend on runtime events.""" + +import json +from pathlib import Path +from typing import Any +from urllib.parse import urlencode + +import pytest + +from webarena_verified import WebArenaVerified + + +def _entry( + *, + method: str, + url: str, + post_data: dict[str, Any], + status: int, + mime_type: str = "application/json", + request_headers: dict[str, str] | None = None, +) -> dict[str, Any]: + body = json.dumps(post_data) if mime_type == "application/json" else urlencode(post_data, doseq=True) + return { + "startedDateTime": "2026-01-01T00:00:00.000Z", + "time": 1, + "request": { + "method": method, + "url": url, + "headers": [{"name": name, "value": value} for name, value in (request_headers or {}).items()], + "cookies": [], + "queryString": [], + "postData": {"mimeType": mime_type, "text": body}, + }, + "response": { + "status": status, + "headers": [], + "cookies": [], + "content": {"size": 0, "mimeType": "application/json", "text": "{}"}, + "redirectURL": "", + }, + "cache": {}, + "timings": {"send": 0, "wait": 1, "receive": 0}, + } + + +def _evaluate( + wa: WebArenaVerified, + tmp_path: Path, + task_id: int, + entries: list[dict[str, Any]], + agent_response: dict[str, Any] | None = None, +): + trace = tmp_path / f"task-{task_id}.har" + trace.write_text( + json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": entries, + } + } + ) + ) + return wa.evaluate_task( + task_id=task_id, + agent_response=agent_response or {"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None}, + network_trace=trace, + ) + + +def test_repeated_endpoint_contracts_match_distinct_request_bodies(wa: WebArenaVerified, tmp_path: Path) -> None: + task = wa.get_task(567) + expected = [item.expected for item in task.network_event_evaluator_cfgs] + assert all(isinstance(item.url, str) for item in expected) + entries = [ + _entry( + method=item.http_method, + url=str(item.url).replace("__GITLAB__", "http://localhost:8023"), + post_data=dict(item.post_data or {}), + status=item.response_status, + ) + for item in expected + ] + + assert float(_evaluate(wa, tmp_path, 567, entries).score) == 1.0 + assert float(_evaluate(wa, tmp_path, 567, entries[1:]).score) == 0.0 + + +def test_hollister_mass_status_uses_the_native_filtered_grid_request(wa: WebArenaVerified, tmp_path: Path) -> None: + correct = _entry( + method="POST", + url="http://localhost:7780/admin/catalog/product/massStatus/status/1/", + post_data={"excluded": "false", "search": "hollister", "namespace": "product_listing"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + unrelated_upstream_contract = _entry( + method="GET", + url="http://localhost:7780/admin/catalog/product/save/id/126/type/configurable/store/0/set/4/back/edit", + post_data={"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + later_wrong_mass_status = _entry( + method="POST", + url="http://localhost:7780/admin/catalog/product/massStatus/status/1/", + post_data={"excluded": "false", "search": "nike", "namespace": "product_listing"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + + assert float(_evaluate(wa, tmp_path, 423, [correct]).score) == 1.0 + assert float(_evaluate(wa, tmp_path, 423, [unrelated_upstream_contract]).score) == 0.0 + assert float(_evaluate(wa, tmp_path, 423, [correct, later_wrong_mass_status]).score) == 0.0 + + +@pytest.mark.parametrize( + ("task_id", "origin", "destination", "cookie"), + [ + (265, "-71.0579762,42.3603713", "-68.2177005,44.3494709", ""), + (266, "-70.2545299,43.6599147", "-68.2177005,44.3494709", ""), + (267, "-68.767507,44.8030715", "-68.2177005,44.3494709", ""), + ( + 268, + "-68.8315387,44.0478975", + "-68.2177005,44.3494709", + "_osm_directions_engine=fossgis_osrm_bicycle", + ), + (759, "-71.060511,42.3554334", "-74.0060152,40.7127281", ""), + (760, "-75.4716115,40.6022552", "-74.4041622,40.0757384", ""), + ], +) +def test_route_contracts_require_intent_direction( + wa: WebArenaVerified, + tmp_path: Path, + task_id: int, + origin: str, + destination: str, + cookie: str, +) -> None: + expected_agent_response = wa.get_task(task_id).expected_agent_response.model_dump(mode="json") + + def route_entry(first: str, second: str) -> dict[str, Any]: + return _entry( + method="GET", + url=f"http://localhost:3000/osrm/routed/route/v1/driving/{first};{second}?steps=true", + post_data={}, + status=200, + request_headers={"Cookie": cookie}, + ) + + def route_trace(first: str, second: str) -> list[dict[str, Any]]: + route = route_entry(first, second) + if not wa.get_task(task_id).is_navigate_task: + return [route] + navigation = _entry( + method="GET", + url="http://localhost:3000/directions", + post_data={}, + status=200, + request_headers={"Accept": "text/html", "Sec-Fetch-Dest": "document"}, + ) + return [navigation, route] + + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace(origin, destination), + expected_agent_response, + ).score + ) + == 1.0 + ) + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace(destination, origin), + expected_agent_response, + ).score + ) + == 0.0 + ) + + +@pytest.mark.parametrize("task_id", [742, 743, 745, 746]) +def test_project_creation_with_multiple_members_matches_complete_trace( + wa: WebArenaVerified, tmp_path: Path, task_id: int +) -> None: + task = wa.get_task(task_id) + entries = [] + for config in task.network_event_evaluator_cfgs: + expected = config.expected + url = ( + "http://localhost:8023/api/v4/projects" + if expected.url == "__GITLAB__/api/v4/projects" + else "http://localhost:8023/api/v4/projects/4242/members" + ) + entries.append( + _entry( + method=expected.http_method, + url=url, + post_data=dict(expected.post_data or {}), + status=expected.response_status, + ) + ) + + assert float(_evaluate(wa, tmp_path, task_id, entries).score) == 1.0 + + +@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:9999", 1.0), ("http://wrong.test", 0.0)]) +def test_markdown_form_values_derender_embedded_urls( + wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float +) -> None: + task = wa.get_task(562) + project, commit = [item.expected for item in task.network_event_evaluator_cfgs] + assert project.post_data is not None + assert commit.post_data is not None + content = commit.post_data["$.actions[0].content"].replace("__REDDIT__", origin) + entries = [ + _entry( + method="POST", + url="http://localhost:8023/projects", + post_data={key: value for key, value in project.post_data.items() if value is not None}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects/byteblaze%2FAwesome_DIY_ideas/repository/commits", + post_data={ + "branch": "main", + "actions": [{"action": "create", "file_path": "README.md", "content": content}], + }, + status=201, + ), + ] + assert float(_evaluate(wa, tmp_path, 562, entries).score) == expected_score + + +@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:8023", 1.0), ("http://wrong.test", 0.0)]) +def test_plain_form_values_derender_embedded_urls( + wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float +) -> None: + expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected + assert expected.post_data is not None + post_data = dict(expected.post_data) + post_data["submission[url]"] = post_data["submission[url]"].replace("__GITLAB__", origin) + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/LifeProTips", + post_data=post_data, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + ] + assert float(_evaluate(wa, tmp_path, 684, entries).score) == expected_score + + +def test_last_event_only_rejects_a_later_wrong_request_to_the_same_endpoint( + wa: WebArenaVerified, tmp_path: Path +) -> None: + expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected + assert expected.post_data is not None + correct = dict(expected.post_data) + correct["submission[url]"] = correct["submission[url]"].replace("__GITLAB__", "http://localhost:8023") + wrong = {**correct, "submission[title]": "wrong title"} + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/LifeProTips", + post_data=post_data, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + for post_data in (correct, wrong) + ] + + assert float(_evaluate(wa, tmp_path, 684, entries).score) == 0.0 + + +def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVerified, tmp_path: Path) -> None: + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/books", + post_data={"submission[title]": "Harry Potter", "submission[forum]": "10037"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/1111/-/comment", + post_data={"reply_to_submission_2222[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + ] + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 1.0 + + entries.append( + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "wrong final comment"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + ) + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0 + + entries.pop() + entries[2]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"}) + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0 + + +def test_dynamic_post_binding_does_not_split_last_event_stream(wa: WebArenaVerified, tmp_path: Path) -> None: + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/MachineLearning", + post_data={ + "submission[title]": "what is the SOTA web navigation agent repo", + "submission[forum]": "10037", + }, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/submit/MachineLearning", + post_data={"submission[title]": "wrong final title", "submission[forum]": "99999"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + ] + + assert float(_evaluate(wa, tmp_path, 604, entries).score) == 0.0 + + +def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebArenaVerified, tmp_path: Path) -> None: + config = wa.get_task(811).network_event_evaluator_cfgs[0] + assert config.post_data_schema == { + "type": "object", + "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}}, + } + entries = [ + _entry( + method="PUT", + url="http://localhost:8023/a11yproject/a11yproject.com/-/issues/1478.json", + post_data={"issue": {"assignee_ids": [2330]}}, + status=200, + ) + ] + assert float(_evaluate(wa, tmp_path, 811, entries).score) == 1.0 + + entries[0]["request"]["postData"]["text"] = json.dumps({"issue": {"assignee_ids": ["2330"]}}) + assert float(_evaluate(wa, tmp_path, 811, entries).score) == 0.0 diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py index 12c2f64..97a7844 100644 --- a/tests/dataset/test_known_contract_corrections.py +++ b/tests/dataset/test_known_contract_corrections.py @@ -39,3 +39,168 @@ def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives( assert post_data["customer_group_ids[0]"] == "1" assert "website_ids" not in post_data assert "customer_group_ids" not in post_data + + for task_id in (699, 700): + task = dataset_by_task_id[task_id] + post_data = _network_expectations(task)[0]["post_data"] + assert task["revision"] == 4 + assert [post_data[f"customer_group_ids[{index}]"] for index in range(3)] == [ + "1", + "2", + "3", + ] + + +def test_mass_status_contract_targets_the_filtered_magento_mutation( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[423] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["last_event_only"] is True + assert evaluator["navigation_only"] is False + assert evaluator["expected"] == { + "url": r"^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$", + "http_method": "POST", + "post_data": { + "excluded": "false", + "search": "hollister", + "namespace": "product_listing", + }, + "response_status": 302, + } + + +def test_existing_color_options_require_only_the_new_product_variations( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + expected = { + 547: ("Phoebe Zipper Sweatshirt", ["size: s, color: brown"]), + 548: ("Frankie Sweatshirt", ["size: s, color: blue", "size: m, color: blue"]), + } + matrix_key = "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes" + for task_id, (product_name, variations) in expected.items(): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 3 + assert len(evaluators) == 1 + assert "product_attribute/save" not in evaluators[0]["expected"]["url"] + assert evaluators[0]["expected"]["post_data"]["product[name]"] == product_name + assert evaluators[0]["expected"]["post_data"][matrix_key] == variations + + +def test_new_size_options_use_the_native_configurable_product_endpoint( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id, label in ((549, "XXXL"), (550, "XXS")): + task = dataset_by_task_id[task_id] + evaluator = next( + item + for item in task["eval"] + if item["evaluator"] == "NetworkEventEvaluator" and "createOptions" in item["expected"]["url"] + ) + assert task["revision"] == 3 + assert evaluator["last_event_only"] is True + assert evaluator["navigation_only"] is False + assert evaluator["ignored_query_params"] == ["isAjax"] + assert evaluator["expected"] == { + "url": r"^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", + "http_method": "POST", + "response_status": 200, + "post_data": { + "options[0][label]": label, + "options[0][attribute_id]": "144", + "options[0][is_new]": "true", + }, + } + + +def test_readme_commit_targets_the_created_repository( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[566] + commit = _network_expectations(task)[1] + assert task["revision"] == 3 + assert commit["url"] == "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits" + + +def test_order_grid_status_contracts_evaluate_the_xhr( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (677, 678, 679, 680): + task = dataset_by_task_id[task_id] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["navigation_only"] is False + assert "mui/index/render/" in evaluator["expected"]["url"] + + +def test_route_contracts_follow_the_intent_origin_to_destination( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + coordinates = { + 265: "-71.0579762,42.3603713;-68.2177005,44.3494709", + 266: "-70.2545299,43.6599147;-68.2177005,44.3494709", + 267: "-68.767507,44.8030715;-68.2177005,44.3494709", + 268: "-68.8315387,44.0478975;-68.2177005,44.3494709", + 737: "-79.9427192,40.4441897;-75.1718916,39.9011873", + 738: "-79.9427192,40.4441897;-75.1712951,39.9042046", + 739: "-79.9427192,40.4441897;-73.9265212,40.8295828", + 740: "-79.9427192,40.4441897;-73.9935443,40.7505085", + 741: "-79.9427192,40.4441897;-71.0621475,42.3662922", + 759: "-71.060511,42.3554334;-74.0060152,40.7127281", + 760: "-75.4716115,40.6022552;-74.4041622,40.0757384", + } + for task_id, coordinate_pair in coordinates.items(): + task = dataset_by_task_id[task_id] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == (5 if task_id in {265, 266, 267, 268} else 4 if task_id in {759, 760} else 3) + if task_id not in {265, 266, 267, 268}: + assert evaluator["navigation_only"] is False + assert coordinate_pair in evaluator["expected"]["url"] + + +def test_replay_specific_changes_do_not_weaken_canonical_contracts( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (448, 744): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 2 + assert all(item.get("last_event_only", True) is True for item in evaluators) + + cart_totals = _network_expectations(dataset_by_task_id[510])[1] + assert cart_totals["response_content"]["items_qty"] == 1 + + +def test_repeated_member_requests_explicitly_match_any_event( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (567, 568, 569, 570): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 3 + assert all(item["last_event_only"] is False for item in evaluators) + + for task_id in (742, 743, 745, 746): + task = dataset_by_task_id[task_id] + member_evaluators = [ + item + for item in task["eval"] + if item["evaluator"] == "NetworkEventEvaluator" and item["expected"]["url"].endswith("/members$") + ] + assert task["revision"] == 3 + assert len(member_evaluators) > 1 + assert all(item["last_event_only"] is False for item in member_evaluators) + + +def test_issue_assignee_array_has_an_explicit_integer_contract( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[811] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["post_data_schema"] == { + "type": "object", + "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}}, + }