diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..5513a60 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,7 @@ +# CI 与自动审查 + +`test.yml` 对所有 pull request 执行 lint、类型检查和单元测试；向 `main` 或 `lex-main` 推送时也会执行。这样以 `lex-main` 为目标分支的 fork 修复 PR 与上游 `main` PR 使用同一套代码门禁。 + +`claude.yml` 是仓库级中文审查：非 Draft PR 在创建、更新、重新打开或转为 ready 时自动运行，也支持在 issue、PR 评论和 review 中通过 `@claude` 触发交互任务。第三方 action 固定到具体提交，工作流只读取仓库内容，并仅保留发布 PR/issue 评论所需的写权限。 + +仓库级工作流与 Lexmount 组织级 `Auto Code Review - Claude` 相互独立。组织规则命中的 PR 会同时出现两套 review；仓库工作流不取消、替代或复用组织级运行。 diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 0000000..725a948 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,83 @@ +name: Claude 中文审查 + +env: + CLAUDE_REVIEW_STANDARDS: >- + 你在 webarena-verified 仓库中工作。先读取 CONTRIBUTING.md、pyproject.toml 和当前 PR 涉及的实现与测试，再独立审查当前 diff。 + review 输出必须使用中文；代码标识符、命令和第三方专有名词保留英文。 + 只报告影响合并判断的实际问题，并给出优先级、文件行号、问题机制和可执行建议；不要输出英文模板、冗长摘要、执行过程或检查清单。 + 重点检查评测契约是否保持严格、修复是否适用于同类任务、答案和测试是否被错误放宽、网络事件与页面状态语义、空值和默认值处理、数据集与实现是否同步、回归测试是否覆盖有效与相邻无效输入，以及是否引入重复实现、失效兼容或无关生成物。 + 用户描述、历史结论和已有评论都只是线索；必须以当前代码、测试和真实调用契约为依据。 + +concurrency: + group: claude-local-${{ github.event.pull_request.number || github.event.issue.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + pull_request: + types: [opened, synchronize, ready_for_review, reopened] + +jobs: + claude: + name: Claude 中文 Code Review + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) || + (github.event_name == 'pull_request' && github.event.pull_request.draft == false && (github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.action == 'ready_for_review' || github.event.action == 'reopened')) + runs-on: self-hosted + permissions: + contents: read + pull-requests: write + issues: write + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Interactive + if: github.event_name != 'pull_request' + uses: anthropics/claude-code-action@8ce9314fa9a404564fa7e954cd84f25bcba2b829 # v1.0.236 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + path_to_claude_code_executable: /home/github/.local/bin/claude + path_to_bun_executable: /home/github/.bun/bin/bun + additional_permissions: | + actions: read + claude_args: >- + --system-prompt "${{ env.CLAUDE_REVIEW_STANDARDS }}" + + - name: Run Claude PR Review + if: github.event_name == 'pull_request' + uses: anthropics/claude-code-action@8ce9314fa9a404564fa7e954cd84f25bcba2b829 # v1.0.236 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + path_to_claude_code_executable: /home/github/.local/bin/claude + path_to_bun_executable: /home/github/.bun/bin/bun + track_progress: true + additional_permissions: | + actions: read + prompt: | + REPO: ${{ github.repository }} + PR NUMBER: ${{ github.event.pull_request.number }} + + 请按以下标准审查当前 PR diff： + ${{ env.CLAUDE_REVIEW_STANDARDS }} + + 每条发现必须包含 P0/P1/P2、文件行号、问题和建议。只报告影响合并判断的问题；没有问题时明确说明残余风险。 + Only post GitHub comments - don't submit review text as messages. + claude_args: | + --system-prompt "${{ env.CLAUDE_REVIEW_STANDARDS }}" + --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 582fc48..540e02b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,13 +2,11 @@ name: Test on: pull_request: - branches: [main] push: - branches: [main] + branches: [main, lex-main] permissions: contents: read - actions: write concurrency: group: test-${{ github.ref }} diff --git a/assets/dataset/webarena-verified.json b/assets/dataset/webarena-verified.json index fe1af37..07a222c 100644 --- a/assets/dataset/webarena-verified.json +++ b/assets/dataset/webarena-verified.json @@ -6722,10 +6722,10 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-71.0579762,42.3603713.*$"} + "expected": {"url": "^.*/route/v1/.*/-71.05777777777777,42.36027777777778;-68.21666666666667,44.35.*$"} } ], - "revision": 4 + "revision": 6 }, { "sites": ["wikipedia", "map"], @@ -6762,10 +6762,10 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-68.2177005,44.3494709;-70.2545299,43.6599147.*$"} + "expected": {"url": "^.*/route/v1/.*/-70.255,43.66;-68.21666666666667,44.35.*$"} } ], - "revision": 4 + "revision": 6 }, { "sites": ["wikipedia", "map"], @@ -6803,12 +6803,12 @@ "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/.*/-68.2177005,44.3494709;-68.767507,44.8030715.*$", + "url": "^.*/route/.*/-68.7675,44.80305555555555;-68.21666666666667,44.35.*$", "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"} } } ], - "revision": 4 + "revision": 6 }, { "sites": ["wikipedia", "map"], @@ -6846,12 +6846,12 @@ "evaluator": "NetworkEventEvaluator", "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/v1/.*/-68.2177005,44.3494709;-68.8315387,44.0478975.*$", + "url": "^.*/route/v1/.*/-68.83166666666666,44.04805555555556;-68.21666666666667,44.35.*$", "headers": {"Cookie": "^.*_osm_directions_engine=fossgis_osrm_bicycle.*$"} } } ], - "revision": 4 + "revision": 6 }, { "sites": ["shopping"], @@ -10821,15 +10821,17 @@ }, { "evaluator": "NetworkEventEvaluator", - "last_event_only": false, + "last_event_only": true, + "navigation_only": false, "expected": { - "url": "^__SHOPPING_ADMIN__/catalog/product/save/id/126/type/configurable/store/0/set/\\d+/back/edit$", - "post_data": {"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"}, + "url": "^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$", + "http_method": "POST", + "post_data": {"excluded": "false", "search": "hollister", "namespace": "product_listing"}, "response_status": 302 } } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -14576,17 +14578,6 @@ "results_schema": {"type": "null"}, "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null} }, - { - "evaluator": "NetworkEventEvaluator", - "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93", - "http_method": "POST", - "response_status": 302, - "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Brown.*$" - } - } - }, { "evaluator": "NetworkEventEvaluator", "post_data_schema": { @@ -14606,7 +14597,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14626,17 +14617,6 @@ "results_schema": {"type": "null"}, "expected": {"task_type": "mutate", "status": "SUCCESS", "retrieved_data": null} }, - { - "evaluator": "NetworkEventEvaluator", - "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/93", - "http_method": "POST", - "response_status": 302, - "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=Blue.*$" - } - } - }, { "evaluator": "NetworkEventEvaluator", "post_data_schema": { @@ -14650,13 +14630,13 @@ "http_method": "POST", "response_status": 302, "post_data": { - "product[name]": "Frankie Sweatshirt", + "product[name]": "Frankie Sweatshirt", "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes": ["size: s, color: blue", "size: m, color: blue"] } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14678,12 +14658,17 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": true, + "navigation_only": false, + "ignored_query_params": ["isAjax"], "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144", + "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", "http_method": "POST", - "response_status": 302, + "response_status": 200, "post_data": { - "serialized_options": "^.*swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXXL.*$" + "options[0][label]": "XXXL", + "options[0][attribute_id]": "144", + "options[0][is_new]": "true" } } }, @@ -14706,7 +14691,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -14728,12 +14713,17 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": true, + "navigation_only": false, + "ignored_query_params": ["isAjax"], "expected": { - "url": "__SHOPPING_ADMIN__/catalog/product_attribute/save/attribute_id/144", + "url": "^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", "http_method": "POST", - "response_status": 302, + "response_status": 200, "post_data": { - "serialized_options": "^.*&swatchtext%5Bvalue%5D%5Boption_\\d+%5D%5B0%5D=XXS.*$" + "options[0][label]": "XXS", + "options[0][attribute_id]": "144", + "options[0][is_new]": "true" } } }, @@ -14756,7 +14746,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -15512,7 +15502,7 @@ "properties": { "$.actions[0].content": {"type": "string", "format": "markdown"} } }, "expected": { - "url": "__GITLAB__//api/v4/projects/byteblaze%2Fdo-it-myself-with/repository/commits", + "url": "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits", "http_method": "POST", "post_data": { "branch": "main", @@ -15525,7 +15515,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15546,6 +15536,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/184/members", "http_method": "POST", @@ -15555,6 +15546,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/184/members", "http_method": "POST", @@ -15563,7 +15555,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15581,6 +15573,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/174/members", "http_method": "POST", @@ -15590,6 +15583,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/174/members", "http_method": "POST", @@ -15598,7 +15592,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15619,6 +15613,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/185/members", "http_method": "POST", @@ -15628,6 +15623,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/185/members", "http_method": "POST", @@ -15636,7 +15632,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -15657,6 +15653,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15666,6 +15663,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15675,6 +15673,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15684,6 +15683,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { "url": "__GITLAB__/api/v4/projects/190/members", "http_method": "POST", @@ -15692,7 +15692,7 @@ } } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping"], @@ -16817,10 +16817,11 @@ "submission[forum]": "{{forum_id}}" }, "response_status": 302 - } + }, + "bindings": { "forum_id": { "allowed_values": ["10043", "10018", "10078"] } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -17036,7 +17037,8 @@ "http_method": "POST", "post_data": {"submission[title]": "Harry Potter", "submission[forum]": "10037"}, "response_status": 302 - } + }, + "event_key": "created_post" }, { "evaluator": "NetworkEventEvaluator", @@ -17045,10 +17047,16 @@ "http_method": "POST", "post_data": {"reply_to_submission_{{post_id}}[comment]": "Wonderful journey"}, "response_status": 302 + }, + "bindings": { + "post_id": { + "source_event": "created_post", + "response_url_pattern": "^__REDDIT__/f/books/(?P\\d+)(?:/.*)?$" + } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -17071,7 +17079,8 @@ "http_method": "POST", "post_data": {"submission[title]": "big little lies", "submission[forum]": "10037"}, "response_status": 302 - } + }, + "event_key": "created_post" }, { "evaluator": "NetworkEventEvaluator", @@ -17080,10 +17089,16 @@ "http_method": "POST", "post_data": {"reply_to_submission_{{post_id}}[comment]": "can't stop it"}, "response_status": 302 + }, + "bindings": { + "post_id": { + "source_event": "created_post", + "response_url_pattern": "^__REDDIT__/f/books/(?P\\d+)(?:/.*)?$" + } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -17106,7 +17121,8 @@ "http_method": "POST", "post_data": {"submission[title]": "Love story", "submission[forum]": "10037"}, "response_status": 302 - } + }, + "event_key": "created_post" }, { "evaluator": "NetworkEventEvaluator", @@ -17115,10 +17131,16 @@ "http_method": "POST", "post_data": {"reply_to_submission_{{post_id}}[comment]": "I cried"}, "response_status": 302 + }, + "bindings": { + "post_id": { + "source_event": "created_post", + "response_url_pattern": "^__REDDIT__/f/books/(?P\\d+)(?:/.*)?$" + } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -17141,7 +17163,8 @@ "http_method": "POST", "post_data": {"submission[title]": "Gone with the wind", "submission[forum]": "10037"}, "response_status": 302 - } + }, + "event_key": "created_post" }, { "evaluator": "NetworkEventEvaluator", @@ -17150,10 +17173,16 @@ "http_method": "POST", "post_data": {"reply_to_submission_{{post_id}}[comment]": "It's a book with history"}, "response_status": 302 + }, + "bindings": { + "post_id": { + "source_event": "created_post", + "response_url_pattern": "^__REDDIT__/f/books/(?P\\d+)(?:/.*)?$" + } } } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit"], @@ -19091,6 +19120,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19105,7 +19135,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19123,6 +19153,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19137,7 +19168,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19155,6 +19186,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "^__SHOPPING_ADMIN__/mui/index/render/.*$", "query_params": { @@ -19170,7 +19202,7 @@ "decode_base64_query": true } ], - "revision": 2 + "revision": 3 }, { "sites": ["shopping_admin"], @@ -19188,6 +19220,7 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "expected": { "url": "__SHOPPING_ADMIN__/mui/index/render/", "headers": {"referer": "__SHOPPING_ADMIN__/sales/order/"}, @@ -19202,7 +19235,7 @@ "ignored_query_params_patterns": ["^paging", "^sorting", "isAjax"] } ], - "revision": 2 + "revision": 3 }, { "sites": ["reddit", "gitlab"], @@ -19862,13 +19895,15 @@ "name": "spring sale", "website_ids[0]": "1", "customer_group_ids[0]": "1", + "customer_group_ids[1]": "2", + "customer_group_ids[2]": "3", "simple_action": "by_percent", "discount_amount": 20 } } } ], - "revision": 3 + "revision": 4 }, { "sites": ["shopping_admin"], @@ -19894,13 +19929,15 @@ "name": "fall discount", "website_ids[0]": "1", "customer_group_ids[0]": "1", + "customer_group_ids[1]": "2", + "customer_group_ids[2]": "3", "simple_action": "cart_fixed", "discount_amount": 10 } } } ], - "revision": 3 + "revision": 4 }, { "sites": ["shopping_admin"], @@ -21374,11 +21411,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-75.1718916,39.9011873;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1718916,39.9011873.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21400,11 +21438,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-75.1712951,39.9042046;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-75.1712951,39.9042046.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21426,11 +21465,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-73.9265212,40.8295828;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9265212,40.8295828.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21452,11 +21492,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-73.9935443,40.7505085;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-73.9935443,40.7505085.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["wikipedia", "map"], @@ -21478,11 +21519,12 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], - "expected": {"url": "^.*/route/v1/.*/-71.0621475,42.3662922;-79.9427192,40.4441897.*$"} + "expected": {"url": "^.*/route/v1/.*/-79.9427192,40.4441897;-71.0621475,42.3662922.*$"} } ], - "revision": 2 + "revision": 3 }, { "sites": ["gitlab"], @@ -21509,28 +21551,37 @@ "http_method": "POST", "post_data": {"name": "planner", "visibility": "private"}, "response_status": 201 - } + }, + "event_key": "created_project" }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 5, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 278, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["gitlab"], @@ -21557,28 +21608,37 @@ "http_method": "POST", "post_data": {"name": "web_arena", "visibility": "public"}, "response_status": 201 - } + }, + "event_key": "created_project" }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 5, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 278, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["gitlab"], @@ -21644,37 +21704,50 @@ "http_method": "POST", "post_data": {"name": "awesome-llms", "visibility": "public"}, "response_status": 201 - } + }, + "event_key": "created_project" }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 2367, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 43, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 5, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["gitlab"], @@ -21701,37 +21774,50 @@ "http_method": "POST", "post_data": {"name": "llm_bulk_inference", "visibility": "private"}, "response_status": 201 - } + }, + "event_key": "created_project" }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 2367, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 43, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } }, { "evaluator": "NetworkEventEvaluator", + "last_event_only": false, "expected": { - "url": "^__GITLAB__/api/v4/projects/\\d+/members$", + "url": "__GITLAB__/api/v4/projects/{{project_id}}/members", "http_method": "POST", "post_data": {"user_id": 5, "access_level": 30}, "response_status": 201 + }, + "bindings": { + "project_id": {"source_event": "created_project", "response_json_path": "$.id"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["gitlab"], @@ -22218,14 +22304,15 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/v1/.*/-71.060511,42.3554334;-1.4869496,55.0252998.*$", + "url": "^.*/route/v1/.*/-71.060511,42.3554334;-74.0060152,40.7127281.*$", "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"} } } ], - "revision": 2 + "revision": 4 }, { "sites": ["map", "shopping_admin"], @@ -22247,14 +22334,15 @@ }, { "evaluator": "NetworkEventEvaluator", + "navigation_only": false, "ignored_query_params_patterns": [".*"], "expected": { - "url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.4041622,40.0757384.*$", + "url": "^.*/route/v1/.*/-75.4716115,40.6022552;-74.0323752,40.7433066.*$", "headers": {"Cookie": "^(?!.*_osm_directions_engine=fossgis_osrm_(?:bicycle|foot)).*$"} } } ], - "revision": 2 + "revision": 6 }, { "sites": ["map"], @@ -23951,6 +24039,12 @@ { "evaluator": "NetworkEventEvaluator", "ignored_query_params": ["serializer"], + "post_data_schema": { + "type": "object", + "properties": { + "$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}} + } + }, "expected": { "url": "__GITLAB__/a11yproject/a11yproject.com/-/issues/1478.json", "http_method": "PUT", @@ -23958,6 +24052,6 @@ } } ], - "revision": 2 + "revision": 3 } ] diff --git a/src/webarena_verified/core/evaluation/data_types/base.py b/src/webarena_verified/core/evaluation/data_types/base.py index dc97022..ef34ec6 100644 --- a/src/webarena_verified/core/evaluation/data_types/base.py +++ b/src/webarena_verified/core/evaluation/data_types/base.py @@ -244,10 +244,7 @@ def _normalize_string(self, s: str) -> str: # Step 0: Try to derender URLs (best effort, no exceptions) # Let the derender_url_fct decide whether to process the string - if self.derender_url_fct is not None and "http" in s: - with contextlib.suppress(Exception): - s = self.derender_url_fct(s, strict=False) - # If derendering fails for any reason, continue with original string + s = self._derender_urls(s) # Step 1: Remove common non-semantic marks BEFORE NFKC normalization # U+2122 ™, U+00AE ®, U+2120 ℠, U+00A9 © @@ -286,3 +283,13 @@ def _normalize_string(self, s: str) -> str: # Step 5: Casefold for robust case-insensitive comparison return s.casefold() + + def _derender_urls(self, value: str) -> str: + """Replace deployment URLs through the configured best-effort callback.""" + if self.derender_url_fct is None or "http" not in value: + return value + with contextlib.suppress(Exception): + # Evaluators bind the desired strictness when they construct this + # callback. Passing it again breaks functools.partial callbacks. + return self.derender_url_fct(value) + return value diff --git a/src/webarena_verified/core/evaluation/data_types/markdown_string.py b/src/webarena_verified/core/evaluation/data_types/markdown_string.py index 34a0054..1ce4f73 100644 --- a/src/webarena_verified/core/evaluation/data_types/markdown_string.py +++ b/src/webarena_verified/core/evaluation/data_types/markdown_string.py @@ -40,7 +40,7 @@ def _normalize_string(self, s: str) -> str: We only strip leading/trailing whitespace. """ - return s.strip() + return self._derender_urls(s).strip() def _type_normalize(self, value: Any) -> str: """Normalize markdown string for comparison. diff --git a/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py index d0486ad..291f8df 100644 --- a/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py +++ b/src/webarena_verified/core/evaluation/evaluators/agent_response_evaluator.py @@ -91,7 +91,11 @@ def _normalized_actual_value( return value _normalized_values = {} - for k in config.expected.model_fields_set: + # Normalize every field in the canonical expected response. Pydantic's + # ``model_fields_set`` omits defaulted ``retrieved_data`` for tasks + # whose dataset contract expects no data, even though the normalized + # expected value contains that field as ``None``. + for k in normalized_expected: if k == "task_type" and k not in value and "performed_operation" in value: k = "performed_operation" # Support legacy field name diff --git a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py index 32c17c2..43ab03e 100644 --- a/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py +++ b/src/webarena_verified/core/evaluation/evaluators/network_event_evaluator.py @@ -5,16 +5,18 @@ """ import re +from collections.abc import Mapping from functools import partial +from itertools import product from types import MappingProxyType from typing import TYPE_CHECKING, Any, cast -from urllib.parse import parse_qs, urlsplit +from urllib.parse import parse_qs, urljoin, urlsplit from webarena_verified.core.evaluation.data_types import URL from webarena_verified.core.utils import logger from webarena_verified.core.utils.jsonpath_utils import extract_jsonpath_value, is_jsonpath_key -from webarena_verified.types.eval import EvalAssertion, EvalStatus, TaskEvalContext -from webarena_verified.types.task import NetworkEventEvaluatorCfg, NetworkEventSpec +from webarena_verified.types.eval import EvalAssertion, EvalStatus, EvaluatorResult, TaskEvalContext +from webarena_verified.types.task import NetworkEventBinding, NetworkEventEvaluatorCfg, NetworkEventSpec from webarena_verified.types.tracing import NetworkEvent from .base import BaseEvaluator @@ -23,6 +25,67 @@ from webarena_verified.types.common import SerializableMappingProxyType +_PLACEHOLDER = re.compile(r"\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}") + + +def _template_match(template: str, actual: Any, bindings: dict[str, str]) -> bool: + """Match a dynamic evaluator template and consistently bind its names.""" + if not isinstance(actual, (str, int, float)): + return False + cursor = 0 + parts = ["^"] + local_groups: set[str] = set() + for match in _PLACEHOLDER.finditer(template): + parts.append(re.escape(template[cursor : match.start()])) + name = match.group(1) + if name in bindings: + parts.append(re.escape(bindings[name])) + elif name in local_groups: + parts.append(f"(?P={name})") + else: + parts.append(f"(?P<{name}>[^/?#&]+)") + local_groups.add(name) + cursor = match.end() + parts.extend((re.escape(template[cursor:]), "$")) + matched = re.fullmatch("".join(parts), str(actual)) + if matched is None: + return False + bindings.update(matched.groupdict()) + return True + + +def _placeholder_names(value: Any) -> set[str]: + if isinstance(value, str): + return set(_PLACEHOLDER.findall(value)) + if isinstance(value, Mapping): + names: set[str] = set() + for key, item in value.items(): + names.update(_placeholder_names(key)) + names.update(_placeholder_names(item)) + return names + if isinstance(value, (list, tuple)): + names = set() + for item in value: + names.update(_placeholder_names(item)) + return names + return set() + + +def _substitute_placeholders(value: Any, bindings: Mapping[str, str]) -> Any: + if isinstance(value, str): + return _PLACEHOLDER.sub(lambda match: bindings[match.group(1)], value) + if isinstance(value, list): + return [_substitute_placeholders(item, bindings) for item in value] + if isinstance(value, tuple): + return tuple(_substitute_placeholders(item, bindings) for item in value) + if isinstance(value, Mapping): + return { + _substitute_placeholders(key, bindings): _substitute_placeholders(item, bindings) + for key, item in value.items() + } + return value + + class NetworkEventEvaluator(BaseEvaluator[NetworkEventEvaluatorCfg]): """Validates network events using four-step architecture. @@ -256,6 +319,16 @@ def _normalize_post_data_field( if post_data_dict: post_data_schema = config.post_data_schema or {} + if ( + not strict + and self._contains_array_schema(post_data_schema) + and not self._matches_json_schema(post_data_dict, post_data_schema) + ): + # Preserve the mismatched raw value so structural comparison + # rejects it instead of coercing (for example, "2330" to 2330). + normalized["post_data"] = post_data_dict + return + _derender_url_fct = partial(context.config.derender_url, sites=context.task.sites, strict=strict) normalized["post_data"] = self.value_normalizer.normalize( post_data_dict, post_data_schema, strict=strict, derender_url_fct=_derender_url_fct @@ -695,3 +768,282 @@ def _should_ignore_post_param(self, param_name: str, config: NetworkEventEvaluat continue return False + + def evaluate(self, *, context: TaskEvalContext, config: NetworkEventEvaluatorCfg) -> EvaluatorResult: + """Evaluate each runtime-bound form of a dynamic network contract.""" + candidates = self._runtime_configs(context, config) + if not candidates: + return super().evaluate(context=context, config=config) + + first_result: EvaluatorResult | None = None + for candidate, source_events in candidates: + candidate_context = context + if source_events is not None: + candidate_context = context.model_copy( + update={"network_trace": context.network_trace.model_copy(update={"events": source_events})} + ) + result = super().evaluate(context=candidate_context, config=candidate) + first_result = first_result or result + if result.score == 1: + return result + assert first_result is not None + return first_result + + def _runtime_configs( + self, context: TaskEvalContext, config: NetworkEventEvaluatorCfg + ) -> list[tuple[NetworkEventEvaluatorCfg, tuple[NetworkEvent, ...] | None]]: + """Resolve declared placeholders and group ordered events by binding.""" + expected = config.expected.model_dump(mode="python", exclude_none=False) + names = _placeholder_names(expected) + if not names: + return [(config, None)] + bindings = config.bindings or {} + if set(bindings) != names: + logger.error( + "Dynamic network contract for task %s must declare one trusted binding per placeholder", + context.task.task_id, + ) + return [] + trusted_bindings = self._trusted_binding_candidates(context, bindings) + if not trusted_bindings: + return [] + return self._dynamic_runtime_configs(context, config, expected, trusted_bindings) + + def _dynamic_runtime_configs( + self, + context: TaskEvalContext, + config: NetworkEventEvaluatorCfg, + expected: dict[str, Any], + trusted_bindings: list[tuple[dict[str, str], int]], + ) -> list[tuple[NetworkEventEvaluatorCfg, tuple[NetworkEvent, ...] | None]]: + """Match ordered consumer events against already trusted binding values.""" + candidates_by_binding: dict[ + tuple[tuple[str, str], ...], tuple[NetworkEventEvaluatorCfg, tuple[tuple[str, str], ...]] + ] = {} + events_by_stream: dict[tuple[tuple[str, str], ...], list[NetworkEvent]] = {} + last_binding_key: tuple[tuple[str, str], ...] | None = None + raw_urls = expected.get("url") + post_templates = expected.get("post_data") or {} + for event_index, event in enumerate(context.network_trace.evaluation_events): + expected_method = expected.get("http_method") + if expected_method and event.http_method.lower() != expected_method.lower(): + continue + + matched_endpoint = False + for trusted, source_index in trusted_bindings: + if source_index >= event_index: + continue + stream = self._resolve_event_stream( + event=event, + context=context, + config=config, + raw_urls=raw_urls, + bindings=trusted, + ) + if stream is None: + continue + if config.last_event_only and not matched_endpoint: + # The last matching endpoint owns the verdict even when a + # different allowed binding matched correctly earlier. + last_binding_key = None + matched_endpoint = True + events_by_stream.setdefault(stream, []).append(event) + + resolved = self._bind_post_placeholders(post_templates, event.post_data or {}, trusted) + if resolved is None: + continue + corrected_expected = _substitute_placeholders(expected, resolved) + binding_key = tuple(sorted(resolved.items())) + if config.last_event_only: + last_binding_key = binding_key + candidates_by_binding.setdefault( + binding_key, + ( + config.model_copy(update={"expected": config.expected.model_copy(update=corrected_expected)}), + stream, + ), + ) + + if config.last_event_only: + candidate_items = [candidates_by_binding[last_binding_key]] if last_binding_key is not None else [] + else: + candidate_items = list(candidates_by_binding.values()) + return [(candidate, tuple(events_by_stream[stream])) for candidate, stream in candidate_items] + + def _trusted_binding_candidates( + self, context: TaskEvalContext, bindings: Mapping[str, NetworkEventBinding] + ) -> list[tuple[dict[str, str], int]]: + """Resolve bindings from closed values or successful earlier event responses.""" + choices: list[list[tuple[str, int, str | None]]] = [] + names: list[str] = [] + for name, binding in bindings.items(): + names.append(name) + if binding.allowed_values: + choices.append([(value, -1, None) for value in binding.allowed_values]) + continue + choices.append( + [ + (value, index, binding.source_event) + for value, index in self._response_binding_values(context, binding) + ] + ) + if any(not values for values in choices): + return [] + candidates: list[tuple[dict[str, str], int]] = [] + for selected in product(*choices): + source_indexes: dict[str, int] = {} + for _, index, source_event in selected: + if source_event is None: + continue + existing = source_indexes.setdefault(source_event, index) + if existing != index: + break + else: + candidates.append( + ( + dict(zip(names, (value for value, _, _ in selected), strict=True)), + max(index for _, index, _ in selected), + ) + ) + return candidates + + def _response_binding_values( + self, context: TaskEvalContext, binding: NetworkEventBinding + ) -> list[tuple[str, int]]: + source_configs = [ + item for item in context.task.network_event_evaluator_cfgs if item.event_key == binding.source_event + ] + if len(source_configs) != 1: + logger.error( + "Task %s binding source %r resolved to %s events", + context.task.task_id, + binding.source_event, + len(source_configs), + ) + return [] + source_config = source_configs[0] + values: list[tuple[str, int]] = [] + events = context.network_trace.evaluation_events + matching_events = self._filter_events_by_criteria(events, context, source_config) + if source_config.last_event_only and matching_events: + matching_events = (matching_events[-1],) + selected_ids = {id(event) for event in matching_events} + for index, event in enumerate(events): + if id(event) not in selected_ids: + continue + source_context = context.model_copy( + update={"network_trace": context.network_trace.model_copy(update={"events": (event,)})} + ) + if super().evaluate(context=source_context, config=source_config).score != 1: + continue + value = self._extract_response_binding(context, event, binding) + if value is not None: + values.append((value, index)) + return values + + @staticmethod + def _extract_response_binding( + context: TaskEvalContext, event: NetworkEvent, binding: NetworkEventBinding + ) -> str | None: + if binding.response_json_path is not None: + content = event.response_content + if content is None: + return None + value = extract_jsonpath_value(content, binding.response_json_path, strict=False) + return str(value) if isinstance(value, (str, int, float)) and not isinstance(value, bool) else None + + assert binding.response_url_pattern is not None + redirect_url = event.redirect_url + if redirect_url is None: + return None + absolute_url = urljoin(event.url, redirect_url) + derendered_url = context.config.derender_url(absolute_url, sites=context.task.sites, strict=False) + if not isinstance(derendered_url, str): + return None + matched = re.fullmatch(binding.response_url_pattern, derendered_url) + return matched.group("value") if matched is not None else None + + def _resolve_event_stream( + self, + *, + event: NetworkEvent, + context: TaskEvalContext, + config: NetworkEventEvaluatorCfg, + raw_urls: Any, + bindings: Mapping[str, str], + ) -> tuple[tuple[str, str], ...] | None: + """Apply trusted URL bindings and the evaluator's normal prefilter.""" + resolved_urls = _substitute_placeholders(raw_urls, bindings) + + filter_config = config.model_copy( + update={"expected": config.expected.model_copy(update={"url": resolved_urls})} + ) + if not self._filter_events_by_criteria((event,), context, filter_config): + return None + return tuple(sorted(bindings.items())) + + @staticmethod + def _bind_post_placeholders( + post_templates: Mapping[str, Any], actual_post: Mapping[str, Any], initial: Mapping[str, str] + ) -> dict[str, str] | None: + """Bind dynamic POST keys and values without defining an event stream.""" + bindings = dict(initial) + for key_template, value_template in post_templates.items(): + actual_value = actual_post.get(key_template) + if _PLACEHOLDER.search(key_template): + matched_key = next( + (key for key in actual_post if _template_match(key_template, key, bindings.copy())), + None, + ) + if matched_key is None or not _template_match(key_template, matched_key, bindings): + return None + actual_value = actual_post[matched_key] + if ( + isinstance(value_template, str) + and _PLACEHOLDER.search(value_template) + and not _template_match(value_template, actual_value, bindings) + ): + return None + return bindings + + @staticmethod + def _matches_json_schema(value: Any, schema: Mapping[str, Any]) -> bool: + """Check the JSON types used by explicit POST-data schemas.""" + schema_type = schema.get("type") + if schema_type == "null": + matches = value is None + elif schema_type == "boolean": + matches = isinstance(value, bool) + elif schema_type == "integer": + matches = isinstance(value, int) and not isinstance(value, bool) + elif schema_type == "number": + matches = isinstance(value, (int, float)) and not isinstance(value, bool) + elif schema_type == "string": + matches = isinstance(value, str) + elif schema_type == "array": + item_schema = schema.get("items", {}) + matches = isinstance(value, (list, tuple)) and all( + NetworkEventEvaluator._matches_json_schema(item, item_schema) for item in value + ) + elif schema_type == "object": + if isinstance(value, Mapping): + matches = all( + key in value and NetworkEventEvaluator._matches_json_schema(value[key], property_schema) + for key, property_schema in schema.get("properties", {}).items() + ) + else: + matches = False + else: + matches = True + return matches + + @staticmethod + def _contains_array_schema(schema: Mapping[str, Any]) -> bool: + """Return whether a schema contains an array at any depth.""" + if schema.get("type") == "array": + return True + properties = schema.get("properties", {}) + return isinstance(properties, Mapping) and any( + isinstance(property_schema, Mapping) and NetworkEventEvaluator._contains_array_schema(property_schema) + for property_schema in properties.values() + ) diff --git a/src/webarena_verified/core/utils/network_event_utils.py b/src/webarena_verified/core/utils/network_event_utils.py index 63e61a8..f88941d 100644 --- a/src/webarena_verified/core/utils/network_event_utils.py +++ b/src/webarena_verified/core/utils/network_event_utils.py @@ -151,7 +151,7 @@ def load_har_trace(har_path: Path) -> tuple["NetworkEvent", ...]: Raises: FileNotFoundError: If HAR file does not exist - ValueError: If HAR format is invalid or contains no entries + ValueError: If HAR format is invalid """ from webarena_verified.types.tracing import NetworkEvent # noqa: PLC0415 (circular import) @@ -167,8 +167,8 @@ def load_har_trace(har_path: Path) -> tuple["NetworkEvent", ...]: raise ValueError("Invalid HAR format: missing 'log.entries' field") entries = har_data["log"]["entries"] - if not entries: - raise ValueError("HAR file contains no entries") + if not isinstance(entries, list): + raise ValueError("Invalid HAR format: 'log.entries' must be a list") # Convert HAR entries to NetworkEvent objects # HAR entries already have the correct structure with request/response fields diff --git a/src/webarena_verified/types/task.py b/src/webarena_verified/types/task.py index fffdfd4..6cf6168 100644 --- a/src/webarena_verified/types/task.py +++ b/src/webarena_verified/types/task.py @@ -1,5 +1,6 @@ """Data models for WebArena Verified tasks (version >= 2.0.0).""" +import re from enum import StrEnum from typing import Annotated, Any, Generic, Literal, Self, TypeVar @@ -187,6 +188,40 @@ class NetworkEventSpec(BaseModel): """ +class NetworkEventBinding(BaseModel): + """Trusted source for a runtime placeholder in a network contract. + + A binding is either selected from a closed set of allowed values or extracted + from the successful response of a named earlier network contract. + """ + + allowed_values: tuple[NonEmptyStr, ...] | None = None + source_event: NonEmptyStr | None = None + response_json_path: NonEmptyStr | None = None + response_url_pattern: NonEmptyStr | None = None + + model_config = ConfigDict(frozen=True, extra="forbid") + + @model_validator(mode="after") + def check_source(self) -> Self: + """Require one unambiguous trusted source mode.""" + response_sources = [self.response_json_path, self.response_url_pattern] + if self.allowed_values: + if self.source_event is not None or any(response_sources): + raise ValueError("allowed_values cannot be combined with a response source") + return self + if self.source_event is None or sum(value is not None for value in response_sources) != 1: + raise ValueError( + "a response binding requires source_event and exactly one of " + "response_json_path or response_url_pattern" + ) + if self.response_url_pattern is not None: + pattern = re.compile(self.response_url_pattern) + if "value" not in pattern.groupindex: + raise ValueError("response_url_pattern must contain a named 'value' capture group") + return self + + class NetworkEventEvaluatorCfg(BaseEval[NetworkEventSpec]): """Validates network events by checking URL, headers, query params, status, event type, and method. @@ -225,6 +260,12 @@ class NetworkEventEvaluatorCfg(BaseEval[NetworkEventSpec]): evaluator: Literal["NetworkEventEvaluator"] = "NetworkEventEvaluator" + event_key: NonEmptyStr | None = None + """Optional task-local name for this event when later contracts bind to its response.""" + + bindings: dict[NonEmptyStr, NetworkEventBinding] | None = None + """Trusted sources for placeholders used by this contract.""" + last_event_only: bool = True """If True, validate only the last matching event. If False, validate if ANY event matches.""" @@ -312,9 +353,18 @@ class WebArenaVerifiedTask(BaseModel): @model_validator(mode="after") def check_eval_has_agent_response(self) -> Self: - """Validate that eval contains at least one AgentResponseEval item.""" + """Validate response coverage and task-local network binding references.""" if not any(isinstance(item, AgentResponseEvaluatorCfg) for item in self.eval): raise ValueError("eval must contain at least one AgentResponseEval item") + network_configs = self.network_event_evaluator_cfgs + event_keys = [item.event_key for item in network_configs if item.event_key is not None] + if len(event_keys) != len(set(event_keys)): + raise ValueError("network evaluator event_key values must be unique within a task") + known_event_keys = set(event_keys) + for config in network_configs: + for binding in (config.bindings or {}).values(): + if binding.source_event is not None and binding.source_event not in known_event_keys: + raise ValueError(f"network binding references unknown source_event {binding.source_event!r}") return self @property diff --git a/tests/api/test_books_description_reference.py b/tests/api/test_books_description_reference.py index a15ff3a..ebd319a 100644 --- a/tests/api/test_books_description_reference.py +++ b/tests/api/test_books_description_reference.py @@ -6,8 +6,9 @@ import pytest from webarena_verified.api import WebArenaVerified -from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig from webarena_verified.types.eval import EvalStatus +from webarena_verified.types.task import WebArenaSite @pytest.mark.parametrize( @@ -26,25 +27,44 @@ ), ], ) -def test_books_description_reference( - task_id: int, answer: list, accepted: bool, tmp_path: Path -) -> None: +def test_books_description_reference(task_id: int, answer: list, accepted: bool, tmp_path: Path) -> None: trace = tmp_path / "network.har" - trace.write_text(json.dumps({"log": {"version": "1.2", "creator": {"name": "pytest", "version": "1"}, "entries": [{ - "startedDateTime": "2026-01-01T00:00:00.000Z", "time": 1, - "request": {"method": "GET", "url": "http://localhost:9999/f/books", "headers": [], "cookies": [], "queryString": []}, - "response": {"status": 200, "headers": [], "cookies": [], "content": {"size": 0, "mimeType": "text/html", "text": ""}, "redirectURL": ""}, - "cache": {}, "timings": {"send": 0, "wait": 1, "receive": 0}, - }]}})) + trace.write_text( + json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": [ + { + "startedDateTime": "2026-01-01T00:00:00.000Z", + "time": 1, + "request": { + "method": "GET", + "url": "http://localhost:9999/f/books", + "headers": [], + "cookies": [], + "queryString": [], + }, + "response": { + "status": 200, + "headers": [], + "cookies": [], + "content": {"size": 0, "mimeType": "text/html", "text": ""}, + "redirectURL": "", + }, + "cache": {}, + "timings": {"send": 0, "wait": 1, "receive": 0}, + } + ], + } + } + ) + ) evaluator = WebArenaVerified( config=WebArenaVerifiedConfig( test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json", - environments={ - "__REDDIT__": { - "urls": ["http://localhost:9999"], - "active_url_idx": 0, - } - }, + environments={WebArenaSite.REDDIT: EnvironmentConfig(urls=["http://localhost:9999"])}, ) ) result = evaluator.evaluate_task( diff --git a/tests/api/test_corrected_task_contracts.py b/tests/api/test_corrected_task_contracts.py index d096350..7a7969c 100644 --- a/tests/api/test_corrected_task_contracts.py +++ b/tests/api/test_corrected_task_contracts.py @@ -7,8 +7,9 @@ import pytest from webarena_verified.api import WebArenaVerified -from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig from webarena_verified.types.eval import EvalStatus +from webarena_verified.types.task import WebArenaSite def _event(url, *, method="GET", form=None, referer=None, navigation=False): @@ -39,7 +40,7 @@ def _event(url, *, method="GET", form=None, referer=None, navigation=False): } -def _evaluate(tmp_path, task, entries, task_type): +def _evaluate(tmp_path, task, entries, task_type, *, status="SUCCESS", retrieved_data=None): path = tmp_path / "network.har" path.write_text( json.dumps({"log": {"version": "1.2", "creator": {"name": "pytest", "version": "1"}, "entries": entries}}) @@ -48,8 +49,9 @@ def _evaluate(tmp_path, task, entries, task_type): config=WebArenaVerifiedConfig( test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json", environments={ - "__GITLAB__": {"urls": ["http://localhost:8023"]}, - "__SHOPPING_ADMIN__": {"urls": ["http://localhost:7780/admin"]}, + WebArenaSite.GITLAB: EnvironmentConfig(urls=["http://localhost:8023"]), + WebArenaSite.SHOPPING: EnvironmentConfig(urls=["http://localhost:7770"]), + WebArenaSite.SHOPPING_ADMIN: EnvironmentConfig(urls=["http://localhost:7780/admin"]), }, ) ) @@ -58,8 +60,8 @@ def _evaluate(tmp_path, task, entries, task_type): task_id=task, agent_response={ "task_type": task_type, - "status": "SUCCESS", - "retrieved_data": None, + "status": status, + "retrieved_data": retrieved_data, }, network_trace=path, ).status @@ -67,6 +69,25 @@ def _evaluate(tmp_path, task, entries, task_type): ) +@pytest.mark.parametrize( + ("retrieved_data", "expected"), + [(None, True), ([0], False)], +) +def test_defaulted_null_retrieved_data_is_still_compared(tmp_path, retrieved_data, expected): + entries = [_event("http://localhost:7770/sales/order/history/", navigation=True)] + assert ( + _evaluate( + tmp_path, + 319, + entries, + "RETRIEVE", + status="NOT_FOUND_ERROR", + retrieved_data=retrieved_data, + ) + is expected + ) + + @pytest.mark.parametrize("left_target", [False, True]) def test_navigation_must_end_on_the_requested_page(tmp_path, left_target): entries = [_event("http://localhost:8023/dashboard/todos", navigation=True)] @@ -104,14 +125,23 @@ def test_empty_project_accepts_unchecked_but_rejects_enabled_readme(tmp_path, re (700, "fall discount", "cart_fixed", 10), ], ) -@pytest.mark.parametrize("customer_group", ["1", "2"]) -def test_price_rule_form_requires_the_requested_customer_group(tmp_path, task, name, action, amount, customer_group): +@pytest.mark.parametrize( + ("customer_groups", "expected"), + [ + (("1", "2", "3"), True), + (("1", "2"), False), + (("1", "2", "4"), False), + ], +) +def test_price_rule_form_requires_all_registered_customer_groups( + tmp_path, task, name, action, amount, customer_groups, expected +): form = { "name": name, "website_ids[0]": "1", - "customer_group_ids[0]": customer_group, "simple_action": action, "discount_amount": amount, } + form.update({f"customer_group_ids[{index}]": value for index, value in enumerate(customer_groups)}) entries = [_event("http://localhost:7780/admin/sales_rule/promo_quote/save/", method="POST", form=form)] - assert _evaluate(tmp_path, task, entries, "MUTATE") is (customer_group == "1") + assert _evaluate(tmp_path, task, entries, "MUTATE") is expected diff --git a/tests/api/test_navigation_xhr_evaluator.py b/tests/api/test_navigation_xhr_evaluator.py index b082455..c7b5d78 100644 --- a/tests/api/test_navigation_xhr_evaluator.py +++ b/tests/api/test_navigation_xhr_evaluator.py @@ -7,8 +7,9 @@ import pytest from webarena_verified.api import WebArenaVerified -from webarena_verified.types.config import WebArenaVerifiedConfig +from webarena_verified.types.config import EnvironmentConfig, WebArenaVerifiedConfig from webarena_verified.types.eval import EvalStatus +from webarena_verified.types.task import WebArenaSite def _entry(url: str, *, referer: str, navigation: bool) -> dict[str, Any]: @@ -41,7 +42,7 @@ def _entry(url: str, *, referer: str, navigation: bool) -> dict[str, Any]: @pytest.mark.parametrize("ending", ["fraud", "notification", "cleared", "left_page", "reloaded"]) def test_navigate_task_checks_the_final_order_grid(tmp_path: Path, ending: str) -> None: base_url = "http://localhost:7780/admin" - trace = { + trace: dict[str, Any] = { "log": { "version": "1.2", "creator": {"name": "pytest", "version": "1"}, @@ -93,12 +94,11 @@ def test_navigate_task_checks_the_final_order_grid(tmp_path: Path, ending: str) config=WebArenaVerifiedConfig( test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json", environments={ - "__SHOPPING_ADMIN__": { - "urls": [base_url], - "active_url_idx": 0, - "use_header_login": True, - "credentials": {"username": "admin", "password": "admin1234"}, - } + WebArenaSite.SHOPPING_ADMIN: EnvironmentConfig( + urls=[base_url], + use_header_login=True, + credentials={"username": "admin", "password": "admin1234"}, + ) }, ) ) @@ -109,3 +109,42 @@ def test_navigate_task_checks_the_final_order_grid(tmp_path: Path, ending: str) ) assert (result.status == EvalStatus.SUCCESS) is (ending in {"fraud", "notification"}) + + +def test_navigate_task_with_empty_har_is_a_normal_failure(tmp_path: Path) -> None: + """A valid empty capture must reach the evaluator instead of crashing as Playwright JSONL.""" + base_url = "http://localhost:7780/admin" + trace_path = tmp_path / "network.har" + trace_path.write_text( + json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": [], + } + } + ) + ) + evaluator = WebArenaVerified( + config=WebArenaVerifiedConfig( + test_data_file=Path(__file__).parents[2] / "assets/dataset/webarena-verified.json", + environments={ + WebArenaSite.SHOPPING_ADMIN: EnvironmentConfig( + urls=[base_url], + use_header_login=True, + credentials={"username": "admin", "password": "admin1234"}, + ) + }, + ) + ) + + result = evaluator.evaluate_task( + task_id=676, + agent_response={"task_type": "NAVIGATE", "status": "SUCCESS", "retrieved_data": None}, + network_trace=trace_path, + ) + + assert result.status == EvalStatus.FAILURE + assert result.score == 0 + assert result.error_msg is None diff --git a/tests/core/evaluation/evaluators/test_network_event_contracts.py b/tests/core/evaluation/evaluators/test_network_event_contracts.py new file mode 100644 index 0000000..c84a7e8 --- /dev/null +++ b/tests/core/evaluation/evaluators/test_network_event_contracts.py @@ -0,0 +1,677 @@ +"""Regression coverage for network contracts that depend on runtime events.""" + +import json +from pathlib import Path +from typing import Any +from urllib.parse import urlencode + +import pytest +from pydantic import ValidationError + +from webarena_verified import WebArenaVerified +from webarena_verified.core.evaluation.evaluators.network_event_evaluator import NetworkEventEvaluator +from webarena_verified.types.eval import TaskEvalContext +from webarena_verified.types.task import NetworkEventBinding +from webarena_verified.types.tracing import NetworkTrace + + +def _entry( + *, + method: str, + url: str, + post_data: dict[str, Any], + status: int, + mime_type: str = "application/json", + request_headers: dict[str, str] | None = None, + response_content: dict[str, Any] | None = None, + redirect_url: str = "", +) -> dict[str, Any]: + body = json.dumps(post_data) if mime_type == "application/json" else urlencode(post_data, doseq=True) + return { + "startedDateTime": "2026-01-01T00:00:00.000Z", + "time": 1, + "request": { + "method": method, + "url": url, + "headers": [{"name": name, "value": value} for name, value in (request_headers or {}).items()], + "cookies": [], + "queryString": [], + "postData": {"mimeType": mime_type, "text": body}, + }, + "response": { + "status": status, + "headers": [], + "cookies": [], + "content": { + "size": 0, + "mimeType": "application/json", + "text": json.dumps(response_content or {}), + }, + "redirectURL": redirect_url, + }, + "cache": {}, + "timings": {"send": 0, "wait": 1, "receive": 0}, + } + + +def _write_trace(tmp_path: Path, task_id: int, entries: list[dict[str, Any]]) -> Path: + trace = tmp_path / f"task-{task_id}.har" + trace.write_text( + json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": entries, + } + } + ) + ) + return trace + + +def _evaluate( + wa: WebArenaVerified, + tmp_path: Path, + task_id: int, + entries: list[dict[str, Any]], + agent_response: dict[str, Any] | None = None, +): + trace = _write_trace(tmp_path, task_id, entries) + return wa.evaluate_task( + task_id=task_id, + agent_response=agent_response or {"task_type": "MUTATE", "status": "SUCCESS", "retrieved_data": None}, + network_trace=trace, + ) + + +def _dynamic_comment_entries() -> list[dict[str, Any]]: + return [ + _entry( + method="POST", + url="http://localhost:9999/submit/books", + post_data={"submission[title]": "Harry Potter", "submission[forum]": "10037"}, + status=302, + mime_type="application/x-www-form-urlencoded", + redirect_url="/f/books/4242/harry-potter", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "wrong final comment"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + ] + + +def test_repeated_endpoint_contracts_match_distinct_request_bodies(wa: WebArenaVerified, tmp_path: Path) -> None: + task = wa.get_task(567) + expected = [item.expected for item in task.network_event_evaluator_cfgs] + assert all(isinstance(item.url, str) for item in expected) + entries = [ + _entry( + method=item.http_method, + url=str(item.url).replace("__GITLAB__", "http://localhost:8023"), + post_data=dict(item.post_data or {}), + status=item.response_status, + ) + for item in expected + ] + + assert float(_evaluate(wa, tmp_path, 567, entries).score) == 1.0 + assert float(_evaluate(wa, tmp_path, 567, entries[1:]).score) == 0.0 + + +def test_hollister_mass_status_uses_the_native_filtered_grid_request(wa: WebArenaVerified, tmp_path: Path) -> None: + correct = _entry( + method="POST", + url="http://localhost:7780/admin/catalog/product/massStatus/status/1/", + post_data={"excluded": "false", "search": "hollister", "namespace": "product_listing"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + unrelated_upstream_contract = _entry( + method="GET", + url="http://localhost:7780/admin/catalog/product/save/id/126/type/configurable/store/0/set/4/back/edit", + post_data={"report_type": "created_at_order", "from": "02/1/2023", "to": "02/28/2023"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + later_wrong_mass_status = _entry( + method="POST", + url="http://localhost:7780/admin/catalog/product/massStatus/status/1/", + post_data={"excluded": "false", "search": "nike", "namespace": "product_listing"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + + assert float(_evaluate(wa, tmp_path, 423, [correct]).score) == 1.0 + assert float(_evaluate(wa, tmp_path, 423, [unrelated_upstream_contract]).score) == 0.0 + assert float(_evaluate(wa, tmp_path, 423, [correct, later_wrong_mass_status]).score) == 0.0 + + +@pytest.mark.parametrize( + ("task_id", "origin", "destination", "cookie", "rejected_destination"), + [ + ( + 265, + "-71.05777777777777,42.36027777777778", + "-68.21666666666667,44.35", + "", + None, + ), + (266, "-70.255,43.66", "-68.21666666666667,44.35", "", None), + ( + 267, + "-68.7675,44.80305555555555", + "-68.21666666666667,44.35", + "", + None, + ), + ( + 268, + "-68.83166666666666,44.04805555555556", + "-68.21666666666667,44.35", + "_osm_directions_engine=fossgis_osrm_bicycle", + None, + ), + (759, "-71.060511,42.3554334", "-74.0060152,40.7127281", "", None), + ( + 760, + "-75.4716115,40.6022552", + "-74.0323752,40.7433066", + "", + "-74.4041622,40.0757384", + ), + ], +) +def test_route_contracts_require_intent_direction( + wa: WebArenaVerified, + tmp_path: Path, + task_id: int, + origin: str, + destination: str, + cookie: str, + rejected_destination: str | None, +) -> None: + expected_agent_response = wa.get_task(task_id).expected_agent_response.model_dump(mode="json") + + def route_entry(first: str, second: str) -> dict[str, Any]: + return _entry( + method="GET", + url=f"http://localhost:3000/osrm/routed/route/v1/driving/{first};{second}?steps=true", + post_data={}, + status=200, + request_headers={"Cookie": cookie}, + ) + + def route_trace(first: str, second: str) -> list[dict[str, Any]]: + route = route_entry(first, second) + if not wa.get_task(task_id).is_navigate_task: + return [route] + navigation = _entry( + method="GET", + url="http://localhost:3000/directions", + post_data={}, + status=200, + request_headers={"Accept": "text/html", "Sec-Fetch-Dest": "document"}, + ) + return [navigation, route] + + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace(origin, destination), + expected_agent_response, + ).score + ) + == 1.0 + ) + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace(destination, origin), + expected_agent_response, + ).score + ) + == 0.0 + ) + if rejected_destination is not None: + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace(origin, rejected_destination), + expected_agent_response, + ).score + ) + == 0.0 + ) + if task_id == 760: + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace( + "-75.44225386838299,40.651163100000005", + destination, + ), + agent_response=expected_agent_response, + ).score + ) + == 0.0 + ) + assert ( + float( + _evaluate( + wa, + tmp_path, + task_id, + route_trace("-75.442,40.651", "-74.032,40.743"), + expected_agent_response, + ).score + ) + == 0.0 + ) + + +@pytest.mark.parametrize("task_id", [742, 743, 745, 746]) +def test_project_creation_with_multiple_members_matches_complete_trace( + wa: WebArenaVerified, tmp_path: Path, task_id: int +) -> None: + task = wa.get_task(task_id) + entries = [] + for config in task.network_event_evaluator_cfgs: + expected = config.expected + url = ( + "http://localhost:8023/api/v4/projects" + if expected.url == "__GITLAB__/api/v4/projects" + else "http://localhost:8023/api/v4/projects/4242/members" + ) + entries.append( + _entry( + method=expected.http_method, + url=url, + post_data=dict(expected.post_data or {}), + status=expected.response_status, + response_content={"id": 4242} if url.endswith("/projects") else None, + ) + ) + + assert float(_evaluate(wa, tmp_path, task_id, entries).score) == 1.0 + + +@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:9999", 1.0), ("http://wrong.test", 0.0)]) +def test_markdown_form_values_derender_embedded_urls( + wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float +) -> None: + task = wa.get_task(562) + project, commit = [item.expected for item in task.network_event_evaluator_cfgs] + assert project.post_data is not None + assert commit.post_data is not None + content = commit.post_data["$.actions[0].content"].replace("__REDDIT__", origin) + entries = [ + _entry( + method="POST", + url="http://localhost:8023/projects", + post_data={key: value for key, value in project.post_data.items() if value is not None}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects/byteblaze%2FAwesome_DIY_ideas/repository/commits", + post_data={ + "branch": "main", + "actions": [{"action": "create", "file_path": "README.md", "content": content}], + }, + status=201, + ), + ] + assert float(_evaluate(wa, tmp_path, 562, entries).score) == expected_score + + +@pytest.mark.parametrize(("origin", "expected_score"), [("http://localhost:8023", 1.0), ("http://wrong.test", 0.0)]) +def test_plain_form_values_derender_embedded_urls( + wa: WebArenaVerified, tmp_path: Path, origin: str, expected_score: float +) -> None: + expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected + assert expected.post_data is not None + post_data = dict(expected.post_data) + post_data["submission[url]"] = post_data["submission[url]"].replace("__GITLAB__", origin) + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/LifeProTips", + post_data=post_data, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + ] + assert float(_evaluate(wa, tmp_path, 684, entries).score) == expected_score + + +def test_last_event_only_rejects_a_later_wrong_request_to_the_same_endpoint( + wa: WebArenaVerified, tmp_path: Path +) -> None: + expected = wa.get_task(684).network_event_evaluator_cfgs[0].expected + assert expected.post_data is not None + correct = dict(expected.post_data) + correct["submission[url]"] = correct["submission[url]"].replace("__GITLAB__", "http://localhost:8023") + wrong = {**correct, "submission[title]": "wrong title"} + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/LifeProTips", + post_data=post_data, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + for post_data in (correct, wrong) + ] + + assert float(_evaluate(wa, tmp_path, 684, entries).score) == 0.0 + + +def test_dynamic_contract_binds_one_value_across_url_and_form_key(wa: WebArenaVerified, tmp_path: Path) -> None: + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/books", + post_data={"submission[title]": "Harry Potter", "submission[forum]": "10037"}, + status=302, + mime_type="application/x-www-form-urlencoded", + redirect_url="/f/books/4242/harry-potter", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/1111/-/comment", + post_data={"reply_to_submission_2222[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + ] + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 1.0 + + entries.append( + _entry( + method="POST", + url="http://localhost:9999/f/books/4242/-/comment", + post_data={"reply_to_submission_4242[comment]": "wrong final comment"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + ) + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0 + + entries.pop() + entries[2]["request"]["postData"]["text"] = urlencode({"reply_to_submission_9999[comment]": "Wonderful journey"}) + assert float(_evaluate(wa, tmp_path, 611, entries).score) == 0.0 + + +def test_dynamic_contract_last_event_only_applies_across_runtime_bindings(wa: WebArenaVerified, tmp_path: Path) -> None: + assert float(_evaluate(wa, tmp_path, 611, _dynamic_comment_entries()).score) == 0.0 + + +def test_dynamic_contract_any_event_mode_accepts_an_earlier_runtime_binding( + wa: WebArenaVerified, tmp_path: Path +) -> None: + task = wa.get_task(611) + comment_config = task.network_event_evaluator_cfgs[1].model_copy(update={"last_event_only": False}) + entries = _dynamic_comment_entries() + + context = TaskEvalContext( + task=task, + agent_response_raw=None, + network_trace=NetworkTrace.from_content(_write_trace(tmp_path, 611, entries)), + config=wa.config, + ) + + assert float(NetworkEventEvaluator().evaluate(context=context, config=comment_config).score) == 1.0 + + +def test_dynamic_post_binding_does_not_split_last_event_stream(wa: WebArenaVerified, tmp_path: Path) -> None: + entries = [ + _entry( + method="POST", + url="http://localhost:9999/submit/MachineLearning", + post_data={ + "submission[title]": "what is the SOTA web navigation agent repo", + "submission[forum]": "10037", + }, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + _entry( + method="POST", + url="http://localhost:9999/submit/MachineLearning", + post_data={"submission[title]": "wrong final title", "submission[forum]": "99999"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ), + ] + + assert float(_evaluate(wa, tmp_path, 604, entries).score) == 0.0 + + +def test_dynamic_forum_binding_accepts_only_the_declared_forum_domain( + wa: WebArenaVerified, tmp_path: Path +) -> None: + def submission(forum_id: str) -> dict[str, Any]: + return _entry( + method="POST", + url="http://localhost:9999/submit", + post_data={ + "submission[title]": "what is the SOTA web navigation agent repo", + "submission[forum]": forum_id, + }, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + + assert float(_evaluate(wa, tmp_path, 604, [submission("10018")]).score) == 1.0 + assert float(_evaluate(wa, tmp_path, 604, [submission("10037")]).score) == 0.0 + + +def test_comment_must_target_the_post_created_by_the_same_task( + wa: WebArenaVerified, tmp_path: Path +) -> None: + created = _entry( + method="POST", + url="http://localhost:9999/submit/books", + post_data={"submission[title]": "Harry Potter", "submission[forum]": "10037"}, + status=302, + mime_type="application/x-www-form-urlencoded", + redirect_url="/f/books/4242/harry-potter", + ) + wrong_post = _entry( + method="POST", + url="http://localhost:9999/f/books/9999/-/comment", + post_data={"reply_to_submission_9999[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + assert float(_evaluate(wa, tmp_path, 611, [created, wrong_post]).score) == 0.0 + + created["response"]["redirectURL"] = "http://evil.test/f/books/9999/forged" + matching_forged_redirect = _entry( + method="POST", + url="http://localhost:9999/f/books/9999/-/comment", + post_data={"reply_to_submission_9999[comment]": "Wonderful journey"}, + status=302, + mime_type="application/x-www-form-urlencoded", + ) + assert float(_evaluate(wa, tmp_path, 611, [created, matching_forged_redirect]).score) == 0.0 + + +@pytest.mark.parametrize("task_id", [742, 743, 745, 746]) +def test_members_must_be_added_to_the_project_created_by_the_same_task( + wa: WebArenaVerified, tmp_path: Path, task_id: int +) -> None: + task = wa.get_task(task_id) + configs = task.network_event_evaluator_cfgs + entries = [ + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects", + post_data=dict(configs[0].expected.post_data or {}), + status=201, + response_content={"id": 4242}, + ) + ] + for index, config in enumerate(configs[1:]): + entries.append( + _entry( + method="POST", + url=f"http://localhost:8023/api/v4/projects/{9000 + index}/members", + post_data=dict(config.expected.post_data or {}), + status=201, + ) + ) + assert float(_evaluate(wa, tmp_path, task_id, entries).score) == 0.0 + + +def test_fields_from_one_source_event_are_bound_atomically(wa: WebArenaVerified, tmp_path: Path) -> None: + task = wa.get_task(742) + creation, member = task.network_event_evaluator_cfgs[:2] + creation = creation.model_copy(update={"last_event_only": False}) + member = member.model_copy( + update={ + "expected": member.expected.model_copy( + update={"url": "__GITLAB__/api/v4/projects/{{project_id}}/namespaces/{{namespace_id}}/members"} + ), + "bindings": { + "project_id": NetworkEventBinding( + source_event="created_project", response_json_path="$.id" + ), + "namespace_id": NetworkEventBinding( + source_event="created_project", response_json_path="$.namespace.id" + ), + }, + } + ) + task = task.model_copy(update={"eval": (task.eval[0], creation, member)}) + + def source(project_id: int, namespace_id: int) -> dict[str, Any]: + return _entry( + method="POST", + url="http://localhost:8023/api/v4/projects", + post_data=dict(creation.expected.post_data or {}), + status=201, + response_content={"id": project_id, "namespace": {"id": namespace_id}}, + ) + + def score(consumer_url: str) -> float: + entries = [ + source(111, 10), + source(222, 20), + _entry( + method="POST", + url=consumer_url, + post_data=dict(member.expected.post_data or {}), + status=201, + ), + ] + context = TaskEvalContext( + task=task, + agent_response_raw=None, + network_trace=NetworkTrace.from_content(_write_trace(tmp_path, 742, entries)), + config=wa.config, + ) + return float(NetworkEventEvaluator().evaluate(context=context, config=member).score) + + assert score("http://localhost:8023/api/v4/projects/222/namespaces/20/members") == 1.0 + assert score("http://localhost:8023/api/v4/projects/111/namespaces/20/members") == 0.0 + + +def test_response_binding_honors_source_last_event_and_order(wa: WebArenaVerified, tmp_path: Path) -> None: + task = wa.get_task(742) + creation, member = task.network_event_evaluator_cfgs[:2] + entries = [ + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects", + post_data=dict(creation.expected.post_data or {}), + status=201, + response_content={"id": 111}, + ), + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects/111/members", + post_data=dict(member.expected.post_data or {}), + status=201, + ), + _entry( + method="POST", + url="http://localhost:8023/api/v4/projects", + post_data=dict(creation.expected.post_data or {}), + status=201, + response_content={"id": 222}, + ), + ] + context = TaskEvalContext( + task=task, + agent_response_raw=None, + network_trace=NetworkTrace.from_content(_write_trace(tmp_path, 742, entries)), + config=wa.config, + ) + assert float(NetworkEventEvaluator().evaluate(context=context, config=member).score) == 0.0 + + +def test_binding_models_reject_mixed_modes_and_duplicate_sources(wa: WebArenaVerified) -> None: + with pytest.raises(ValidationError, match="allowed_values cannot be combined"): + NetworkEventBinding( + allowed_values=("1",), source_event="created_project", response_json_path="$.id" + ) + + task = wa.get_task(742) + duplicate = task.network_event_evaluator_cfgs[0] + task_data = task.model_dump(mode="python") + task_data["eval"] = (*task.eval, duplicate) + with pytest.raises(ValidationError, match="event_key values must be unique"): + type(task).model_validate(task_data) + + +def test_singleton_array_post_contract_is_an_array_not_an_alternative(wa: WebArenaVerified, tmp_path: Path) -> None: + config = wa.get_task(811).network_event_evaluator_cfgs[0] + assert config.post_data_schema == { + "type": "object", + "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}}, + } + entries = [ + _entry( + method="PUT", + url="http://localhost:8023/a11yproject/a11yproject.com/-/issues/1478.json", + post_data={"issue": {"assignee_ids": [2330]}}, + status=200, + ) + ] + assert float(_evaluate(wa, tmp_path, 811, entries).score) == 1.0 + + entries[0]["request"]["postData"]["text"] = json.dumps({"issue": {"assignee_ids": ["2330"]}}) + assert float(_evaluate(wa, tmp_path, 811, entries).score) == 0.0 diff --git a/tests/dataset/test_known_contract_corrections.py b/tests/dataset/test_known_contract_corrections.py index 12c2f64..6b30aac 100644 --- a/tests/dataset/test_known_contract_corrections.py +++ b/tests/dataset/test_known_contract_corrections.py @@ -39,3 +39,208 @@ def test_indexed_form_fields_are_not_encoded_as_singleton_alternatives( assert post_data["customer_group_ids[0]"] == "1" assert "website_ids" not in post_data assert "customer_group_ids" not in post_data + + for task_id in (699, 700): + task = dataset_by_task_id[task_id] + post_data = _network_expectations(task)[0]["post_data"] + assert task["revision"] == 4 + assert [post_data[f"customer_group_ids[{index}]"] for index in range(3)] == [ + "1", + "2", + "3", + ] + + +def test_mass_status_contract_targets_the_filtered_magento_mutation( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[423] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["last_event_only"] is True + assert evaluator["navigation_only"] is False + assert evaluator["expected"] == { + "url": r"^__SHOPPING_ADMIN__/catalog/product/massStatus/status/1/?$", + "http_method": "POST", + "post_data": { + "excluded": "false", + "search": "hollister", + "namespace": "product_listing", + }, + "response_status": 302, + } + + +def test_existing_color_options_require_only_the_new_product_variations( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + expected = { + 547: ("Phoebe Zipper Sweatshirt", ["size: s, color: brown"]), + 548: ("Frankie Sweatshirt", ["size: s, color: blue", "size: m, color: blue"]), + } + matrix_key = "$['configurable-matrix-serialized'][?(@.newProduct == 1)].attributes" + for task_id, (product_name, variations) in expected.items(): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 3 + assert len(evaluators) == 1 + assert "product_attribute/save" not in evaluators[0]["expected"]["url"] + assert evaluators[0]["expected"]["post_data"]["product[name]"] == product_name + assert evaluators[0]["expected"]["post_data"][matrix_key] == variations + + +def test_new_size_options_use_the_native_configurable_product_endpoint( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id, label in ((549, "XXXL"), (550, "XXS")): + task = dataset_by_task_id[task_id] + evaluator = next( + item + for item in task["eval"] + if item["evaluator"] == "NetworkEventEvaluator" and "createOptions" in item["expected"]["url"] + ) + assert task["revision"] == 3 + assert evaluator["last_event_only"] is True + assert evaluator["navigation_only"] is False + assert evaluator["ignored_query_params"] == ["isAjax"] + assert evaluator["expected"] == { + "url": r"^__SHOPPING_ADMIN__/catalog/product_attribute/createOptions/?$", + "http_method": "POST", + "response_status": 200, + "post_data": { + "options[0][label]": label, + "options[0][attribute_id]": "144", + "options[0][is_new]": "true", + }, + } + + +def test_readme_commit_targets_the_created_repository( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[566] + commit = _network_expectations(task)[1] + assert task["revision"] == 3 + assert commit["url"] == "__GITLAB__/api/v4/projects/byteblaze%2Fdo-it-myself/repository/commits" + + +def test_order_grid_status_contracts_evaluate_the_xhr( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (677, 678, 679, 680): + task = dataset_by_task_id[task_id] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["navigation_only"] is False + assert "mui/index/render/" in evaluator["expected"]["url"] + + +def test_route_contracts_follow_the_intent_origin_to_destination( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + coordinates = { + 265: "-71.05777777777777,42.36027777777778;-68.21666666666667,44.35", + 266: "-70.255,43.66;-68.21666666666667,44.35", + 267: "-68.7675,44.80305555555555;-68.21666666666667,44.35", + 268: "-68.83166666666666,44.04805555555556;-68.21666666666667,44.35", + 737: "-79.9427192,40.4441897;-75.1718916,39.9011873", + 738: "-79.9427192,40.4441897;-75.1712951,39.9042046", + 739: "-79.9427192,40.4441897;-73.9265212,40.8295828", + 740: "-79.9427192,40.4441897;-73.9935443,40.7505085", + 741: "-79.9427192,40.4441897;-71.0621475,42.3662922", + 759: "-71.060511,42.3554334;-74.0060152,40.7127281", + 760: "-75.4716115,40.6022552;-74.0323752,40.7433066", + } + for task_id, coordinate_pair in coordinates.items(): + task = dataset_by_task_id[task_id] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + expected_revision = 6 if task_id in {265, 266, 267, 268, 760} else 4 if task_id == 759 else 3 + assert task["revision"] == expected_revision + if task_id not in {265, 266, 267, 268}: + assert evaluator["navigation_only"] is False + assert coordinate_pair in evaluator["expected"]["url"] + + task_760_url = next( + item for item in dataset_by_task_id[760]["eval"] if item["evaluator"] == "NetworkEventEvaluator" + )["expected"]["url"] + assert "-74.4041622,40.0757384" not in task_760_url + assert "-75.44225386838299,40.651163100000005" not in task_760_url + + +def test_replay_specific_changes_do_not_weaken_canonical_contracts( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (448, 744): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 2 + assert all(item.get("last_event_only", True) is True for item in evaluators) + + cart_totals = _network_expectations(dataset_by_task_id[510])[1] + assert cart_totals["response_content"]["items_qty"] == 1 + + +def test_repeated_member_requests_explicitly_match_any_event( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + for task_id in (567, 568, 569, 570): + task = dataset_by_task_id[task_id] + evaluators = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 3 + assert all(item["last_event_only"] is False for item in evaluators) + + for task_id in (742, 743, 745, 746): + task = dataset_by_task_id[task_id] + member_evaluators = [ + item + for item in task["eval"] + if item["evaluator"] == "NetworkEventEvaluator" and item["expected"]["url"].endswith("/members") + ] + creation = next( + item + for item in task["eval"] + if item["evaluator"] == "NetworkEventEvaluator" + and item["expected"]["url"] == "__GITLAB__/api/v4/projects" + ) + assert task["revision"] == 4 + assert creation["event_key"] == "created_project" + assert len(member_evaluators) > 1 + assert all(item["last_event_only"] is False for item in member_evaluators) + assert all( + item["bindings"] + == {"project_id": {"source_event": "created_project", "response_json_path": "$.id"}} + for item in member_evaluators + ) + + +def test_dynamic_network_values_have_explicit_trusted_sources( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task_604 = dataset_by_task_id[604] + forum = next(item for item in task_604["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task_604["revision"] == 3 + assert forum["bindings"] == {"forum_id": {"allowed_values": ["10043", "10018", "10078"]}} + + for task_id in (611, 612, 613, 614): + task = dataset_by_task_id[task_id] + creation, comment = [item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator"] + assert task["revision"] == 3 + assert creation["event_key"] == "created_post" + assert comment["bindings"] == { + "post_id": { + "source_event": "created_post", + "response_url_pattern": r"^__REDDIT__/f/books/(?P\d+)(?:/.*)?$", + } + } + + +def test_issue_assignee_array_has_an_explicit_integer_contract( + dataset_by_task_id: dict[int, dict[str, Any]], +) -> None: + task = dataset_by_task_id[811] + evaluator = next(item for item in task["eval"] if item["evaluator"] == "NetworkEventEvaluator") + assert task["revision"] == 3 + assert evaluator["post_data_schema"] == { + "type": "object", + "properties": {"$.issue.assignee_ids": {"type": "array", "items": {"type": "integer"}}}, + } diff --git a/tests/types/test_network_trace.py b/tests/types/test_network_trace.py index 8bbeefa..839303b 100644 --- a/tests/types/test_network_trace.py +++ b/tests/types/test_network_trace.py @@ -1,5 +1,7 @@ """Functional tests for NetworkTrace.""" +import json + import pytest from webarena_verified.types.tracing import NetworkTrace @@ -27,6 +29,29 @@ def test_from_har_loads_successfully(test_har_path): assert len(trace.evaluation_events) > 0 +def test_empty_har_remains_a_valid_har_trace(tmp_path): + """An empty HAR is evidence of no observed requests, not another trace format.""" + trace_path = tmp_path / "network.har" + trace_path.write_text( + json.dumps( + { + "log": { + "version": "1.2", + "creator": {"name": "pytest", "version": "1"}, + "entries": [], + } + } + ) + ) + + trace = NetworkTrace.from_content(trace_path) + + assert trace.is_playwright is False + assert trace.src_file == trace_path + assert trace.events == () + assert trace.evaluation_events == () + + @pytest.mark.skip(reason="Skipped to focus on other tests") def test_trace_properties(trace): """Test basic NetworkTrace properties."""